diff options
Diffstat (limited to 'Jellyfin.Server')
52 files changed, 3004 insertions, 1158 deletions
diff --git a/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs b/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs index 6066893de..a56baba33 100644 --- a/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs +++ b/Jellyfin.Server/Extensions/ApiApplicationBuilderExtensions.cs @@ -69,16 +69,6 @@ namespace Jellyfin.Server.Extensions } /// <summary> - /// Adds LAN based access filtering to the application pipeline. - /// </summary> - /// <param name="appBuilder">The application builder.</param> - /// <returns>The updated application builder.</returns> - public static IApplicationBuilder UseLanFiltering(this IApplicationBuilder appBuilder) - { - return appBuilder.UseMiddleware<LanFilteringMiddleware>(); - } - - /// <summary> /// Enables url decoding before binding to the application pipeline. /// </summary> /// <param name="appBuilder">The <see cref="IApplicationBuilder"/>.</param> diff --git a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs index c6c3f21fe..09a4e2ed3 100644 --- a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs +++ b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs @@ -119,15 +119,15 @@ namespace Jellyfin.Server.Extensions // https://github.com/dotnet/aspnetcore/blob/master/src/Middleware/HttpOverrides/src/ForwardedHeadersMiddleware.cs // Enable debug logging on Microsoft.AspNetCore.HttpOverrides.ForwardedHeadersMiddleware to help investigate issues. - options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost; - if (config.KnownProxies.Length == 0) { + options.ForwardedHeaders = ForwardedHeaders.None; options.KnownNetworks.Clear(); options.KnownProxies.Clear(); } else { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost; AddProxyAddresses(config, config.KnownProxies, options); } @@ -215,7 +215,7 @@ namespace Jellyfin.Server.Extensions }); // Add all xml doc files to swagger generator. - var xmlFiles = Directory.GetFiles( + var xmlFiles = Directory.EnumerateFiles( AppContext.BaseDirectory, "*.xml", SearchOption.TopDirectoryOnly); diff --git a/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs b/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs index 7695c0d9e..be9cf0f15 100644 --- a/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs +++ b/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs @@ -1,10 +1,14 @@ using System; +using System.Collections.Generic; using System.IO; using System.Net; +using System.Security.Cryptography.X509Certificates; using Jellyfin.Server.Helpers; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Extensions; +using MediaBrowser.Model.Net; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -35,56 +39,98 @@ public static class WebHostBuilderExtensions return builder .UseKestrel((builderContext, options) => { - var addresses = appHost.NetManager.GetAllBindInterfaces(false); + SetupJellyfinWebServer( + appHost.NetManager.GetAllBindInterfaces(false), + appHost.HttpPort, + appHost.ListenWithHttps ? appHost.HttpsPort : null, + appHost.Certificate, + startupConfig, + appPaths, + logger, + builderContext, + options); + }) + .UseStartup(context => new Startup(appHost, context.Configuration)); + } - bool flagged = false; - foreach (var netAdd in addresses) + /// <summary> + /// Configures a Kestrel type webServer to bind to the specific arguments. + /// </summary> + /// <param name="addresses">The IP addresses that should be listend to.</param> + /// <param name="httpPort">The http port.</param> + /// <param name="httpsPort">If set the https port. If set you must also set the certificate.</param> + /// <param name="certificate">The certificate used for https port.</param> + /// <param name="startupConfig">The startup config.</param> + /// <param name="appPaths">The app paths.</param> + /// <param name="logger">A logger.</param> + /// <param name="builderContext">The kestrel build pipeline context.</param> + /// <param name="options">The kestrel server options.</param> + /// <exception cref="InvalidOperationException">Will be thrown when a https port is set but no or an invalid certificate is provided.</exception> + public static void SetupJellyfinWebServer( + IReadOnlyList<IPData> addresses, + int httpPort, + int? httpsPort, + X509Certificate2? certificate, + IConfiguration startupConfig, + IApplicationPaths appPaths, + ILogger logger, + WebHostBuilderContext builderContext, + KestrelServerOptions options) + { + bool flagged = false; + foreach (var netAdd in addresses) + { + var address = netAdd.Address; + logger.LogInformation("Kestrel is listening on {Address}", address.Equals(IPAddress.IPv6Any) ? "all interfaces" : address); + options.Listen(netAdd.Address, httpPort); + if (httpsPort.HasValue) + { + if (builderContext.HostingEnvironment.IsDevelopment()) { - var address = netAdd.Address; - logger.LogInformation("Kestrel is listening on {Address}", address.Equals(IPAddress.IPv6Any) ? "all interfaces" : address); - options.Listen(netAdd.Address, appHost.HttpPort); - if (appHost.ListenWithHttps) + try { options.Listen( address, - appHost.HttpsPort, - listenOptions => listenOptions.UseHttps(appHost.Certificate)); + httpsPort.Value, + listenOptions => listenOptions.UseHttps()); } - else if (builderContext.HostingEnvironment.IsDevelopment()) + catch (InvalidOperationException) { - try + if (!flagged) { - options.Listen( - address, - appHost.HttpsPort, - listenOptions => listenOptions.UseHttps()); - } - catch (InvalidOperationException) - { - if (!flagged) - { - logger.LogWarning("Failed to listen to HTTPS using the ASP.NET Core HTTPS development certificate. Please ensure it has been installed and set as trusted"); - flagged = true; - } + logger.LogWarning("Failed to listen to HTTPS using the ASP.NET Core HTTPS development certificate. Please ensure it has been installed and set as trusted"); + flagged = true; } } } - - // Bind to unix socket (only on unix systems) - if (startupConfig.UseUnixSocket() && Environment.OSVersion.Platform == PlatformID.Unix) + else { - var socketPath = StartupHelpers.GetUnixSocketPath(startupConfig, appPaths); - - // Workaround for https://github.com/aspnet/AspNetCore/issues/14134 - if (File.Exists(socketPath)) + if (certificate is null) { - File.Delete(socketPath); + throw new InvalidOperationException("Cannot run jellyfin with https without setting a valid certificate."); } - options.ListenUnixSocket(socketPath); - logger.LogInformation("Kestrel listening to unix socket {SocketPath}", socketPath); + options.Listen( + address, + httpsPort.Value, + listenOptions => listenOptions.UseHttps(certificate)); } - }) - .UseStartup(context => new Startup(appHost, context.Configuration)); + } + } + + // Bind to unix socket (only on unix systems) + if (startupConfig.UseUnixSocket() && Environment.OSVersion.Platform == PlatformID.Unix) + { + var socketPath = StartupHelpers.GetUnixSocketPath(startupConfig, appPaths); + + // Workaround for https://github.com/aspnet/AspNetCore/issues/14134 + if (File.Exists(socketPath)) + { + File.Delete(socketPath); + } + + options.ListenUnixSocket(socketPath); + logger.LogInformation("Kestrel listening to unix socket {SocketPath}", socketPath); + } } } diff --git a/Jellyfin.Server/Filters/AdditionalModelFilter.cs b/Jellyfin.Server/Filters/AdditionalModelFilter.cs index bf38f741c..421eeecda 100644 --- a/Jellyfin.Server/Filters/AdditionalModelFilter.cs +++ b/Jellyfin.Server/Filters/AdditionalModelFilter.cs @@ -25,7 +25,7 @@ namespace Jellyfin.Server.Filters public class AdditionalModelFilter : IDocumentFilter { // Array of options that should not be visible in the api spec. - private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions) }; + private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions), typeof(MediaBrowser.Model.Branding.BrandingOptions) }; private readonly IServerConfigurationManager _serverConfigurationManager; /// <summary> @@ -92,17 +92,51 @@ namespace Jellyfin.Server.Filters continue; } - // Additional discriminator needed for GroupUpdate models... - if (messageType == SessionMessageType.SyncPlayGroupUpdate && type != typeof(SyncPlayGroupUpdateCommandMessage)) - { - continue; - } - var schema = context.SchemaGenerator.GenerateSchema(type, context.SchemaRepository); outboundWebSocketSchemas.Add(schema); outboundWebSocketDiscriminators.Add(messageType.ToString()!, schema.Reference.ReferenceV3); } + // Add custom "SyncPlayGroupUpdateMessage" schema because Swashbuckle cannot generate it for us + var syncPlayGroupUpdateMessageSchema = new OpenApiSchema + { + Type = "object", + Description = "Untyped sync play command.", + Properties = new Dictionary<string, OpenApiSchema> + { + { + "Data", new OpenApiSchema + { + AllOf = + [ + new OpenApiSchema { Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = nameof(GroupUpdate<object>) } } + ], + Description = "Group update data", + Nullable = false, + } + }, + { "MessageId", new OpenApiSchema { Type = "string", Format = "uuid", Description = "Gets or sets the message id." } }, + { + "MessageType", new OpenApiSchema + { + Enum = Enum.GetValues<SessionMessageType>().Select(type => new OpenApiString(type.ToString())).ToList<IOpenApiAny>(), + AllOf = + [ + new OpenApiSchema { Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = nameof(SessionMessageType) } } + ], + Description = "The different kinds of messages that are used in the WebSocket api.", + Default = new OpenApiString(nameof(SessionMessageType.SyncPlayGroupUpdate)), + ReadOnly = true + } + }, + }, + AdditionalPropertiesAllowed = false, + Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = "SyncPlayGroupUpdateMessage" } + }; + context.SchemaRepository.AddDefinition("SyncPlayGroupUpdateMessage", syncPlayGroupUpdateMessageSchema); + outboundWebSocketSchemas.Add(syncPlayGroupUpdateMessageSchema); + outboundWebSocketDiscriminators[nameof(SessionMessageType.SyncPlayGroupUpdate)] = syncPlayGroupUpdateMessageSchema.Reference.ReferenceV3; + var outboundWebSocketMessageSchema = new OpenApiSchema { Type = "object", @@ -140,41 +174,46 @@ namespace Jellyfin.Server.Filters }); // Manually generate sync play GroupUpdate messages. - if (!context.SchemaRepository.Schemas.TryGetValue(nameof(GroupUpdate), out var groupUpdateSchema)) - { - groupUpdateSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate), context.SchemaRepository); - } - - var groupUpdateOfGroupInfoSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<GroupInfoDto>), context.SchemaRepository); - var groupUpdateOfGroupStateSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<GroupStateUpdate>), context.SchemaRepository); - var groupUpdateOfStringSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<string>), context.SchemaRepository); - var groupUpdateOfPlayQueueSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<PlayQueueUpdate>), context.SchemaRepository); + var groupUpdateTypes = typeof(GroupUpdate<>).Assembly.GetTypes() + .Where(t => t.BaseType != null + && t.BaseType.IsGenericType + && t.BaseType.GetGenericTypeDefinition() == typeof(GroupUpdate<>)) + .ToList(); - groupUpdateSchema.OneOf = new List<OpenApiSchema> + var groupUpdateSchemas = new List<OpenApiSchema>(); + var groupUpdateDiscriminators = new Dictionary<string, string>(); + foreach (var type in groupUpdateTypes) { - groupUpdateOfGroupInfoSchema, - groupUpdateOfGroupStateSchema, - groupUpdateOfStringSchema, - groupUpdateOfPlayQueueSchema - }; + var groupUpdateType = (GroupUpdateType?)type.GetProperty(nameof(GroupUpdate<object>.Type))?.GetCustomAttribute<DefaultValueAttribute>()?.Value; + if (groupUpdateType is null) + { + continue; + } + + var schema = context.SchemaGenerator.GenerateSchema(type, context.SchemaRepository); + groupUpdateSchemas.Add(schema); + groupUpdateDiscriminators[groupUpdateType.ToString()!] = schema.Reference.ReferenceV3; + } - groupUpdateSchema.Discriminator = new OpenApiDiscriminator + var groupUpdateSchema = new OpenApiSchema { - PropertyName = nameof(GroupUpdate.Type), - Mapping = new Dictionary<string, string> + Type = "object", + Description = "Represents the list of possible group update types", + Reference = new OpenApiReference + { + Id = nameof(GroupUpdate<object>), + Type = ReferenceType.Schema + }, + OneOf = groupUpdateSchemas, + Discriminator = new OpenApiDiscriminator { - { GroupUpdateType.UserJoined.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.UserLeft.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupJoined.ToString(), groupUpdateOfGroupInfoSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupLeft.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.StateUpdate.ToString(), groupUpdateOfGroupStateSchema.Reference.ReferenceV3 }, - { GroupUpdateType.PlayQueue.ToString(), groupUpdateOfPlayQueueSchema.Reference.ReferenceV3 }, - { GroupUpdateType.NotInGroup.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupDoesNotExist.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.LibraryAccessDenied.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 } + PropertyName = nameof(GroupUpdate<object>.Type), + Mapping = groupUpdateDiscriminators } }; + context.SchemaRepository.Schemas[nameof(GroupUpdate<object>)] = groupUpdateSchema; + context.SchemaGenerator.GenerateSchema(typeof(ServerDiscoveryInfo), context.SchemaRepository); foreach (var configuration in _serverConfigurationManager.GetConfigurationStores()) diff --git a/Jellyfin.Server/Helpers/StartupHelpers.cs b/Jellyfin.Server/Helpers/StartupHelpers.cs index bbf6d31f1..93c996166 100644 --- a/Jellyfin.Server/Helpers/StartupHelpers.cs +++ b/Jellyfin.Server/Helpers/StartupHelpers.cs @@ -3,18 +3,19 @@ using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; -using System.Net; using System.Runtime.InteropServices; using System.Runtime.Versioning; using System.Text; using System.Threading.Tasks; using Emby.Server.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Extensions; using MediaBrowser.Model.IO; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Serilog; +using Serilog.Extensions.Logging; using ILogger = Microsoft.Extensions.Logging.ILogger; namespace Jellyfin.Server.Helpers; @@ -257,11 +258,14 @@ public static class StartupHelpers { try { + var startupLogger = new LoggerProviderCollection(); + startupLogger.AddProvider(new SetupServer.SetupLoggerFactory()); // Serilog.Log is used by SerilogLoggerFactory when no logger is specified Log.Logger = new LoggerConfiguration() .ReadFrom.Configuration(configuration) .Enrich.FromLogContext() .Enrich.WithThreadId() + .WriteTo.Async(e => e.Providers(startupLogger)) .CreateLogger(); } catch (Exception ex) diff --git a/Jellyfin.Server/Jellyfin.Server.csproj b/Jellyfin.Server/Jellyfin.Server.csproj index 452b03efb..14c4285fe 100644 --- a/Jellyfin.Server/Jellyfin.Server.csproj +++ b/Jellyfin.Server/Jellyfin.Server.csproj @@ -48,6 +48,7 @@ <PackageReference Include="Microsoft.Extensions.Configuration.Json" /> <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" /> <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" /> + <PackageReference Include="Morestachio" /> <PackageReference Include="prometheus-net" /> <PackageReference Include="prometheus-net.AspNetCore" /> <PackageReference Include="Serilog.AspNetCore" /> @@ -79,6 +80,9 @@ <None Update="wwwroot\api-docs\banner-dark.svg"> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> </None> + <None Update="ServerSetupApp/index.mstemplate.html"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> </ItemGroup> </Project> diff --git a/Jellyfin.Server/Migrations/IAsyncMigrationRoutine.cs b/Jellyfin.Server/Migrations/IAsyncMigrationRoutine.cs new file mode 100644 index 000000000..5b6a5fe94 --- /dev/null +++ b/Jellyfin.Server/Migrations/IAsyncMigrationRoutine.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Jellyfin.Server.Migrations; + +/// <summary> +/// Interface that describes a migration routine. +/// </summary> +internal interface IAsyncMigrationRoutine +{ + /// <summary> + /// Execute the migration routine. + /// </summary> + /// <param name="cancellationToken">A cancellation token triggered if the migration should be aborted.</param> + /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> + public Task PerformAsync(CancellationToken cancellationToken); +} + +/// <summary> +/// Interface that describes a migration routine. +/// </summary> +[Obsolete("Use IAsyncMigrationRoutine instead")] +internal interface IMigrationRoutine +{ + /// <summary> + /// Execute the migration routine. + /// </summary> + [Obsolete("Use IAsyncMigrationRoutine.PerformAsync instead")] + public void Perform(); +} diff --git a/Jellyfin.Server/Migrations/IDatabaseMigrationRoutine.cs b/Jellyfin.Server/Migrations/IDatabaseMigrationRoutine.cs index 78ff1e3fd..d2d80a81e 100644 --- a/Jellyfin.Server/Migrations/IDatabaseMigrationRoutine.cs +++ b/Jellyfin.Server/Migrations/IDatabaseMigrationRoutine.cs @@ -7,6 +7,8 @@ namespace Jellyfin.Server.Migrations; /// <summary> /// Defines a migration that operates on the Database. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete internal interface IDatabaseMigrationRoutine : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { } diff --git a/Jellyfin.Server/Migrations/IMigrationRoutine.cs b/Jellyfin.Server/Migrations/IMigrationRoutine.cs deleted file mode 100644 index 29f681df5..000000000 --- a/Jellyfin.Server/Migrations/IMigrationRoutine.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using Jellyfin.Server.Implementations; -using Microsoft.EntityFrameworkCore.Internal; - -namespace Jellyfin.Server.Migrations -{ - /// <summary> - /// Interface that describes a migration routine. - /// </summary> - internal interface IMigrationRoutine - { - /// <summary> - /// Gets the unique id for this migration. This should never be modified after the migration has been created. - /// </summary> - public Guid Id { get; } - - /// <summary> - /// Gets the display name of the migration. - /// </summary> - public string Name { get; } - - /// <summary> - /// Gets a value indicating whether to perform migration on a new install. - /// </summary> - public bool PerformOnNewInstall { get; } - - /// <summary> - /// Execute the migration routine. - /// </summary> - public void Perform(); - } -} diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs new file mode 100644 index 000000000..70e54125b --- /dev/null +++ b/Jellyfin.Server/Migrations/JellyfinMigrationAttribute.cs @@ -0,0 +1,68 @@ +#pragma warning disable CA1019 // Define accessors for attribute arguments + +using System; +using System.Globalization; +using Jellyfin.Server.Migrations.Stages; + +namespace Jellyfin.Server.Migrations; + +/// <summary> +/// Declares an class as an migration with its set metadata. +/// </summary> +[AttributeUsage(AttributeTargets.Class, Inherited = true, AllowMultiple = false)] +public sealed class JellyfinMigrationAttribute : Attribute +{ + /// <summary> + /// Initializes a new instance of the <see cref="JellyfinMigrationAttribute"/> class. + /// </summary> + /// <param name="order">The ordering this migration should be applied to. Must be a valid DateTime ISO8601 formatted string.</param> + /// <param name="name">The name of this Migration.</param> +#pragma warning disable CS0618 // Type or member is obsolete + public JellyfinMigrationAttribute(string order, string name) : this(order, name, null) +#pragma warning restore CS0618 // Type or member is obsolete + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JellyfinMigrationAttribute"/> class for legacy migrations. + /// </summary> + /// <param name="order">The ordering this migration should be applied to. Must be a valid DateTime ISO8601 formatted string.</param> + /// <param name="name">The name of this Migration.</param> + /// <param name="key">[ONLY FOR LEGACY MIGRATIONS]The unique key of this migration. Must be a valid Guid formatted string.</param> + [Obsolete("This Constructor should only be used for Legacy migrations. Use the (Order,Name) one for all new ones instead.")] + public JellyfinMigrationAttribute(string order, string name, string? key) + { + Order = DateTime.Parse(order, CultureInfo.InvariantCulture); + Name = name; + Stage = JellyfinMigrationStageTypes.AppInitialisation; + if (key is not null) + { + Key = Guid.Parse(key); + } + } + + /// <summary> + /// Gets or Sets a value indicating whether the annoated migration should be executed on a fresh install. + /// </summary> + public bool RunMigrationOnSetup { get; set; } + + /// <summary> + /// Gets or Sets the stage the annoated migration should be executed at. Defaults to <see cref="JellyfinMigrationStageTypes.CoreInitialisation"/>. + /// </summary> + public JellyfinMigrationStageTypes Stage { get; set; } = JellyfinMigrationStageTypes.CoreInitialisation; + + /// <summary> + /// Gets the ordering of the migration. + /// </summary> + public DateTime Order { get; } + + /// <summary> + /// Gets the name of the migration. + /// </summary> + public string Name { get; } + + /// <summary> + /// Gets the Legacy Key of the migration. Not required for new Migrations. + /// </summary> + public Guid? Key { get; } +} diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs new file mode 100644 index 000000000..6c8da7e82 --- /dev/null +++ b/Jellyfin.Server/Migrations/JellyfinMigrationBackupAttribute.cs @@ -0,0 +1,35 @@ +using System; + +namespace Jellyfin.Server.Migrations; + +/// <summary> +/// Marks an <see cref="JellyfinMigrationAttribute"/> migration and instructs the <see cref="JellyfinMigrationService"/> to perform a backup. +/// </summary> +[AttributeUsage(System.AttributeTargets.Class, Inherited = true, AllowMultiple = true)] +public sealed class JellyfinMigrationBackupAttribute : System.Attribute +{ + /// <summary> + /// Gets or Sets a value indicating whether a backup of the old library.db should be performed. + /// </summary> + public bool LegacyLibraryDb { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Database should be performed. + /// </summary> + public bool JellyfinDb { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the metadata folder should be performed. + /// </summary> + public bool Metadata { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Trickplay folder should be performed. + /// </summary> + public bool Trickplay { get; set; } + + /// <summary> + /// Gets or Sets a value indicating whether a backup of the Subtitles folder should be performed. + /// </summary> + public bool Subtitles { get; set; } +} diff --git a/Jellyfin.Server/Migrations/JellyfinMigrationService.cs b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs new file mode 100644 index 000000000..5331b43e3 --- /dev/null +++ b/Jellyfin.Server/Migrations/JellyfinMigrationService.cs @@ -0,0 +1,451 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.Serialization; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.Implementations.SystemBackupService; +using Jellyfin.Server.Migrations.Stages; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.SystemBackupService; +using MediaBrowser.Model.Configuration; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations; + +/// <summary> +/// Handles Migration of the Jellyfin data structure. +/// </summary> +internal class JellyfinMigrationService +{ + private const string DbFilename = "library.db"; + private readonly IDbContextFactory<JellyfinDbContext> _dbContextFactory; + private readonly ILoggerFactory _loggerFactory; + private readonly IStartupLogger _startupLogger; + private readonly IBackupService? _backupService; + private readonly IJellyfinDatabaseProvider? _jellyfinDatabaseProvider; + private readonly IApplicationPaths _applicationPaths; + private (string? LibraryDb, string? JellyfinDb, BackupManifestDto? FullBackup) _backupKey; + + /// <summary> + /// Initializes a new instance of the <see cref="JellyfinMigrationService"/> class. + /// </summary> + /// <param name="dbContextFactory">Provides access to the jellyfin database.</param> + /// <param name="loggerFactory">The logger factory.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="applicationPaths">Application paths for library.db backup.</param> + /// <param name="backupService">The jellyfin backup service.</param> + /// <param name="jellyfinDatabaseProvider">The jellyfin database provider.</param> + public JellyfinMigrationService( + IDbContextFactory<JellyfinDbContext> dbContextFactory, + ILoggerFactory loggerFactory, + IStartupLogger startupLogger, + IApplicationPaths applicationPaths, + IBackupService? backupService = null, + IJellyfinDatabaseProvider? jellyfinDatabaseProvider = null) + { + _dbContextFactory = dbContextFactory; + _loggerFactory = loggerFactory; + _startupLogger = startupLogger; + _backupService = backupService; + _jellyfinDatabaseProvider = jellyfinDatabaseProvider; + _applicationPaths = applicationPaths; +#pragma warning disable CS0618 // Type or member is obsolete + Migrations = [.. typeof(IMigrationRoutine).Assembly.GetTypes().Where(e => typeof(IMigrationRoutine).IsAssignableFrom(e) || typeof(IAsyncMigrationRoutine).IsAssignableFrom(e)) + .Select(e => (Type: e, Metadata: e.GetCustomAttribute<JellyfinMigrationAttribute>(), Backup: e.GetCustomAttributes<JellyfinMigrationBackupAttribute>())) + .Where(e => e.Metadata != null) + .GroupBy(e => e.Metadata!.Stage) + .Select(f => + { + var stage = new MigrationStage(f.Key); + foreach (var item in f) + { + JellyfinMigrationBackupAttribute? backupMetadata = null; + if (item.Backup?.Any() == true) + { + backupMetadata = item.Backup.Aggregate(MergeBackupAttributes); + } + + stage.Add(new(item.Type, item.Metadata!, backupMetadata)); + } + + return stage; + })]; +#pragma warning restore CS0618 // Type or member is obsolete + } + + private interface IInternalMigration + { + Task PerformAsync(IStartupLogger logger); + } + + private HashSet<MigrationStage> Migrations { get; set; } + + public async Task CheckFirstTimeRunOrMigration(IApplicationPaths appPaths) + { + var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migration Startup"); + logger.LogInformation("Initialise Migration service."); + var xmlSerializer = new MyXmlSerializer(); + var serverConfig = File.Exists(appPaths.SystemConfigurationFilePath) + ? (ServerConfiguration)xmlSerializer.DeserializeFromFile(typeof(ServerConfiguration), appPaths.SystemConfigurationFilePath)! + : new ServerConfiguration(); + if (!serverConfig.IsStartupWizardCompleted) + { + logger.LogInformation("System initialisation detected. Seed data."); + var flatApplyMigrations = Migrations.SelectMany(e => e.Where(f => !f.Metadata.RunMigrationOnSetup)).ToArray(); + + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService<IHistoryRepository>(); + + await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false); + var startupScripts = flatApplyMigrations + .Where(e => !appliedMigrations.Any(f => f != e.BuildCodeMigrationId())) + .Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion())))) + .ToArray(); + foreach (var item in startupScripts) + { + logger.LogInformation("Seed migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name); + await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false); + } + } + + logger.LogInformation("Migration system initialisation completed."); + } + else + { + // migrate any existing migration.xml files + var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, "migrations.xml"); + var migrationOptions = File.Exists(migrationConfigPath) + ? (MigrationOptions)xmlSerializer.DeserializeFromFile(typeof(MigrationOptions), migrationConfigPath)! + : null; + if (migrationOptions != null && migrationOptions.Applied.Count > 0) + { + logger.LogInformation("Old migration style migration.xml detected. Migrate now."); + try + { + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService<IHistoryRepository>(); + var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false); + var lastOldAppliedMigration = Migrations + .SelectMany(e => e.Where(e => e.Metadata.Key is not null)) // only consider migrations that have the key set as its the reference marker for legacy migrations. + .Where(e => migrationOptions.Applied.Any(f => f.Id.Equals(e.Metadata.Key!.Value))) + .Where(e => !appliedMigrations.Contains(e.BuildCodeMigrationId())) + .OrderBy(e => e.BuildCodeMigrationId()) + .Last(); // this is the latest migration applied in the old migration.xml + + IReadOnlyList<CodeMigration> oldMigrations = [ + .. Migrations + .SelectMany(e => e) + .OrderBy(e => e.BuildCodeMigrationId()) + .TakeWhile(e => e.BuildCodeMigrationId() != lastOldAppliedMigration.BuildCodeMigrationId()), + lastOldAppliedMigration + ]; + // those are all migrations that had to run in the old migration system, even if not noted in the migration.xml file. + + var startupScripts = oldMigrations.Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion())))); + foreach (var item in startupScripts) + { + logger.LogInformation("Migrate migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name); + await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false); + } + + logger.LogInformation("Rename old migration.xml to migration.xml.backup"); + File.Move(migrationConfigPath, Path.ChangeExtension(migrationConfigPath, ".xml.backup"), true); + } + } + catch (Exception ex) + { + logger.LogCritical(ex, "Failed to apply migrations"); + throw; + } + } + } + } + + public async Task MigrateStepAsync(JellyfinMigrationStageTypes stage, IServiceProvider? serviceProvider) + { + var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migrate stage {stage}."); + ICollection<CodeMigration> migrationStage = (Migrations.FirstOrDefault(e => e.Stage == stage) as ICollection<CodeMigration>) ?? []; + + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService<IHistoryRepository>(); + var migrationsAssembly = dbContext.GetService<IMigrationsAssembly>(); + var appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + var pendingCodeMigrations = migrationStage + .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) + .Select(e => (Key: e.BuildCodeMigrationId(), Migration: new InternalCodeMigration(e, serviceProvider, dbContext))) + .ToArray(); + + (string Key, InternalDatabaseMigration Migration)[] pendingDatabaseMigrations = []; + if (stage is JellyfinMigrationStageTypes.CoreInitialisation) + { + pendingDatabaseMigrations = migrationsAssembly.Migrations.Where(f => appliedMigrations.All(e => e.MigrationId != f.Key)) + .Select(e => (Key: e.Key, Migration: new InternalDatabaseMigration(e, dbContext))) + .ToArray(); + } + + (string Key, IInternalMigration Migration)[] pendingMigrations = [.. pendingCodeMigrations, .. pendingDatabaseMigrations]; + logger.LogInformation("There are {Pending} migrations for stage {Stage}.", pendingCodeMigrations.Length, stage); + var migrations = pendingMigrations.OrderBy(e => e.Key).ToArray(); + + foreach (var item in migrations) + { + var migrationLogger = logger.With(_loggerFactory.CreateLogger(item.Migration.GetType().Name)).BeginGroup($"{item.Key}"); + try + { + migrationLogger.LogInformation("Perform migration {Name}", item.Key); + await item.Migration.PerformAsync(migrationLogger).ConfigureAwait(false); + migrationLogger.LogInformation("Migration {Name} was successfully applied", item.Key); + } + catch (Exception ex) + { + migrationLogger.LogCritical("Error: {Error}", ex.Message); + migrationLogger.LogError(ex, "Migration {Name} failed", item.Key); + + if (_backupKey != default && _backupService is not null && _jellyfinDatabaseProvider is not null) + { + if (_backupKey.LibraryDb is not null) + { + migrationLogger.LogInformation("Attempt to rollback librarydb."); + try + { + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + File.Move(_backupKey.LibraryDb, libraryDbPath, true); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null) + { + migrationLogger.LogInformation("Attempt to rollback JellyfinDb."); + try + { + await _jellyfinDatabaseProvider.RestoreBackupFast(_backupKey.JellyfinDb, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + migrationLogger.LogInformation("Attempt to rollback from backup."); + try + { + await _backupService.RestoreBackupAsync(_backupKey.FullBackup.Path).ConfigureAwait(false); + } + catch (Exception inner) + { + migrationLogger.LogCritical(inner, "Could not rollback from backup {Backup}. Manual intervention might be required to restore a operational state.", _backupKey.FullBackup.Path); + } + } + } + + throw; + } + } + } + } + + private static string GetJellyfinVersion() + { + return Assembly.GetEntryAssembly()!.GetName().Version!.ToString(); + } + + public async Task CleanupSystemAfterMigration(ILogger logger) + { + if (_backupKey != default) + { + if (_backupKey.LibraryDb is not null) + { + logger.LogInformation("Attempt to cleanup librarydb backup."); + try + { + File.Delete(_backupKey.LibraryDb); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.LibraryDb); + } + } + + if (_backupKey.JellyfinDb is not null && _jellyfinDatabaseProvider is not null) + { + logger.LogInformation("Attempt to cleanup JellyfinDb backup."); + try + { + await _jellyfinDatabaseProvider.DeleteBackup(_backupKey.JellyfinDb).ConfigureAwait(false); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.JellyfinDb); + } + } + + if (_backupKey.FullBackup is not null) + { + logger.LogInformation("Attempt to cleanup from migration backup."); + try + { + File.Delete(_backupKey.FullBackup.Path); + } + catch (Exception inner) + { + logger.LogCritical(inner, "Could not cleanup backup {Backup}.", _backupKey.FullBackup.Path); + } + } + } + } + + public async Task PrepareSystemForMigration(ILogger logger) + { + logger.LogInformation("Prepare system for possible migrations"); + JellyfinMigrationBackupAttribute backupInstruction; + IReadOnlyList<HistoryRow> appliedMigrations; + var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var historyRepository = dbContext.GetService<IHistoryRepository>(); + var migrationsAssembly = dbContext.GetService<IMigrationsAssembly>(); + appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); + backupInstruction = new JellyfinMigrationBackupAttribute() + { + JellyfinDb = migrationsAssembly.Migrations.Any(f => appliedMigrations.All(e => e.MigrationId != f.Key)) + }; + } + + backupInstruction = Migrations.SelectMany(e => e) + .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) + .Select(e => e.BackupRequirements) + .Where(e => e is not null) + .Aggregate(backupInstruction, MergeBackupAttributes!); + + if (backupInstruction.LegacyLibraryDb) + { + logger.LogInformation("A migration will attempt to modify the library.db, will attempt to backup the file now."); + // for legacy migrations that still operates on the library.db + var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); + if (File.Exists(libraryDbPath)) + { + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", libraryDbPath, i); + if (!File.Exists(bakPath)) + { + try + { + logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); + File.Copy(libraryDbPath, bakPath); + _backupKey = (bakPath, _backupKey.JellyfinDb, _backupKey.FullBackup); + logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; + } + } + } + + logger.LogInformation("{Library} has been backed up as {BackupPath}", DbFilename, _backupKey.LibraryDb); + } + else + { + logger.LogError("Cannot make a backup of {Library} at path {BackupPath} because file could not be found at {LibraryPath}", DbFilename, libraryDbPath, _applicationPaths.DataPath); + } + } + + if (backupInstruction.JellyfinDb && _jellyfinDatabaseProvider != null) + { + logger.LogInformation("A migration will attempt to modify the jellyfin.db, will attempt to backup the file now."); + _backupKey = (_backupKey.LibraryDb, await _jellyfinDatabaseProvider.MigrationBackupFast(CancellationToken.None).ConfigureAwait(false), _backupKey.FullBackup); + logger.LogInformation("Jellyfin database has been backed up as {BackupPath}", _backupKey.JellyfinDb); + } + + if (_backupService is not null && (backupInstruction.Metadata || backupInstruction.Subtitles || backupInstruction.Trickplay)) + { + logger.LogInformation("A migration will attempt to modify system resources. Will attempt to create backup now."); + _backupKey = (_backupKey.LibraryDb, _backupKey.JellyfinDb, await _backupService.CreateBackupAsync(new BackupOptionsDto() + { + Metadata = backupInstruction.Metadata, + Subtitles = backupInstruction.Subtitles, + Trickplay = backupInstruction.Trickplay, + Database = false // database backups are explicitly handled by the provider itself as the backup service requires parity with the current model + }).ConfigureAwait(false)); + logger.LogInformation("Pre-Migration backup successfully created as {BackupKey}", _backupKey.FullBackup.Path); + } + } + + private static JellyfinMigrationBackupAttribute MergeBackupAttributes(JellyfinMigrationBackupAttribute left, JellyfinMigrationBackupAttribute right) + { + return new JellyfinMigrationBackupAttribute() + { + JellyfinDb = left!.JellyfinDb || right!.JellyfinDb, + LegacyLibraryDb = left.LegacyLibraryDb || right!.LegacyLibraryDb, + Metadata = left.Metadata || right!.Metadata, + Subtitles = left.Subtitles || right!.Subtitles, + Trickplay = left.Trickplay || right!.Trickplay + }; + } + + private class InternalCodeMigration : IInternalMigration + { + private readonly CodeMigration _codeMigration; + private readonly IServiceProvider? _serviceProvider; + private JellyfinDbContext _dbContext; + + public InternalCodeMigration(CodeMigration codeMigration, IServiceProvider? serviceProvider, JellyfinDbContext dbContext) + { + _codeMigration = codeMigration; + _serviceProvider = serviceProvider; + _dbContext = dbContext; + } + + public async Task PerformAsync(IStartupLogger logger) + { + await _codeMigration.Perform(_serviceProvider, logger, CancellationToken.None).ConfigureAwait(false); + + var historyRepository = _dbContext.GetService<IHistoryRepository>(); + var createScript = historyRepository.GetInsertScript(new HistoryRow(_codeMigration.BuildCodeMigrationId(), GetJellyfinVersion())); + await _dbContext.Database.ExecuteSqlRawAsync(createScript).ConfigureAwait(false); + } + } + + private class InternalDatabaseMigration : IInternalMigration + { + private readonly JellyfinDbContext _jellyfinDbContext; + private KeyValuePair<string, TypeInfo> _databaseMigrationInfo; + + public InternalDatabaseMigration(KeyValuePair<string, TypeInfo> databaseMigrationInfo, JellyfinDbContext jellyfinDbContext) + { + _databaseMigrationInfo = databaseMigrationInfo; + _jellyfinDbContext = jellyfinDbContext; + } + + public async Task PerformAsync(IStartupLogger logger) + { + var migrator = _jellyfinDbContext.GetService<IMigrator>(); + await migrator.MigrateAsync(_databaseMigrationInfo.Key).ConfigureAwait(false); + } + } +} diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs deleted file mode 100644 index 9865199f3..000000000 --- a/Jellyfin.Server/Migrations/MigrationRunner.cs +++ /dev/null @@ -1,181 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Emby.Server.Implementations; -using Emby.Server.Implementations.Serialization; -using Jellyfin.Database.Implementations; -using Jellyfin.Server.Implementations; -using MediaBrowser.Common.Configuration; -using MediaBrowser.Model.Configuration; -using Microsoft.EntityFrameworkCore.Storage; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace Jellyfin.Server.Migrations -{ - /// <summary> - /// The class that knows which migrations to apply and how to apply them. - /// </summary> - public sealed class MigrationRunner - { - /// <summary> - /// The list of known pre-startup migrations, in order of applicability. - /// </summary> - private static readonly Type[] _preStartupMigrationTypes = - { - typeof(PreStartupRoutines.CreateNetworkConfiguration), - typeof(PreStartupRoutines.MigrateMusicBrainzTimeout), - typeof(PreStartupRoutines.MigrateNetworkConfiguration), - typeof(PreStartupRoutines.MigrateEncodingOptions), - typeof(PreStartupRoutines.RenameEnableGroupingIntoCollections) - }; - - /// <summary> - /// The list of known migrations, in order of applicability. - /// </summary> - private static readonly Type[] _migrationTypes = - { - typeof(Routines.DisableTranscodingThrottling), - typeof(Routines.CreateUserLoggingConfigFile), - typeof(Routines.MigrateActivityLogDb), - typeof(Routines.RemoveDuplicateExtras), - typeof(Routines.AddDefaultPluginRepository), - typeof(Routines.MigrateUserDb), - typeof(Routines.ReaddDefaultPluginRepository), - typeof(Routines.MigrateDisplayPreferencesDb), - typeof(Routines.RemoveDownloadImagesInAdvance), - typeof(Routines.MigrateAuthenticationDb), - typeof(Routines.FixPlaylistOwner), - typeof(Routines.MigrateRatingLevels), - typeof(Routines.AddDefaultCastReceivers), - typeof(Routines.UpdateDefaultPluginRepository), - typeof(Routines.FixAudioData), - typeof(Routines.MoveTrickplayFiles), - typeof(Routines.RemoveDuplicatePlaylistChildren), - typeof(Routines.MigrateLibraryDb), - }; - - /// <summary> - /// Run all needed migrations. - /// </summary> - /// <param name="host">CoreAppHost that hosts current version.</param> - /// <param name="loggerFactory">Factory for making the logger.</param> - /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> - public static async Task Run(CoreAppHost host, ILoggerFactory loggerFactory) - { - var logger = loggerFactory.CreateLogger<MigrationRunner>(); - var migrations = _migrationTypes - .Select(m => ActivatorUtilities.CreateInstance(host.ServiceProvider, m)) - .OfType<IMigrationRoutine>() - .ToArray(); - - var migrationOptions = host.ConfigurationManager.GetConfiguration<MigrationOptions>(MigrationsListStore.StoreKey); - HandleStartupWizardCondition(migrations, migrationOptions, host.ConfigurationManager.Configuration.IsStartupWizardCompleted, logger); - await PerformMigrations(migrations, migrationOptions, options => host.ConfigurationManager.SaveConfiguration(MigrationsListStore.StoreKey, options), logger, host.ServiceProvider.GetRequiredService<IJellyfinDatabaseProvider>()) - .ConfigureAwait(false); - } - - /// <summary> - /// Run all needed pre-startup migrations. - /// </summary> - /// <param name="appPaths">Application paths.</param> - /// <param name="loggerFactory">Factory for making the logger.</param> - /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> - public static async Task RunPreStartup(ServerApplicationPaths appPaths, ILoggerFactory loggerFactory) - { - var logger = loggerFactory.CreateLogger<MigrationRunner>(); - var migrations = _preStartupMigrationTypes - .Select(m => Activator.CreateInstance(m, appPaths, loggerFactory)) - .OfType<IMigrationRoutine>() - .ToArray(); - - var xmlSerializer = new MyXmlSerializer(); - var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, MigrationsListStore.StoreKey.ToLowerInvariant() + ".xml"); - var migrationOptions = File.Exists(migrationConfigPath) - ? (MigrationOptions)xmlSerializer.DeserializeFromFile(typeof(MigrationOptions), migrationConfigPath)! - : new MigrationOptions(); - - // We have to deserialize it manually since the configuration manager may overwrite it - var serverConfig = File.Exists(appPaths.SystemConfigurationFilePath) - ? (ServerConfiguration)xmlSerializer.DeserializeFromFile(typeof(ServerConfiguration), appPaths.SystemConfigurationFilePath)! - : new ServerConfiguration(); - - HandleStartupWizardCondition(migrations, migrationOptions, serverConfig.IsStartupWizardCompleted, logger); - await PerformMigrations(migrations, migrationOptions, options => xmlSerializer.SerializeToFile(options, migrationConfigPath), logger, null).ConfigureAwait(false); - } - - private static void HandleStartupWizardCondition(IEnumerable<IMigrationRoutine> migrations, MigrationOptions migrationOptions, bool isStartWizardCompleted, ILogger logger) - { - if (isStartWizardCompleted) - { - return; - } - - // If startup wizard is not finished, this is a fresh install. - var onlyOldInstalls = migrations.Where(m => !m.PerformOnNewInstall).ToArray(); - logger.LogInformation("Marking following migrations as applied because this is a fresh install: {@OnlyOldInstalls}", onlyOldInstalls.Select(m => m.Name)); - migrationOptions.Applied.AddRange(onlyOldInstalls.Select(m => (m.Id, m.Name))); - } - - private static async Task PerformMigrations( - IMigrationRoutine[] migrations, - MigrationOptions migrationOptions, - Action<MigrationOptions> saveConfiguration, - ILogger logger, - IJellyfinDatabaseProvider? jellyfinDatabaseProvider) - { - // save already applied migrations, and skip them thereafter - saveConfiguration(migrationOptions); - var appliedMigrationIds = migrationOptions.Applied.Select(m => m.Id).ToHashSet(); - var migrationsToBeApplied = migrations.Where(e => !appliedMigrationIds.Contains(e.Id)).ToArray(); - - string? migrationKey = null; - if (jellyfinDatabaseProvider is not null && migrationsToBeApplied.Any(f => f is IDatabaseMigrationRoutine)) - { - logger.LogInformation("Performing database backup"); - try - { - migrationKey = await jellyfinDatabaseProvider.MigrationBackupFast(CancellationToken.None).ConfigureAwait(false); - logger.LogInformation("Database backup with key '{BackupKey}' has been successfully created.", migrationKey); - } - catch (NotImplementedException) - { - logger.LogWarning("Could not perform backup of database before migration because provider does not support it"); - } - } - - try - { - foreach (var migrationRoutine in migrationsToBeApplied) - { - logger.LogInformation("Applying migration '{Name}'", migrationRoutine.Name); - - try - { - migrationRoutine.Perform(); - } - catch (Exception ex) - { - logger.LogError(ex, "Could not apply migration '{Name}'", migrationRoutine.Name); - throw; - } - - // Mark the migration as completed - logger.LogInformation("Migration '{Name}' applied successfully", migrationRoutine.Name); - migrationOptions.Applied.Add((migrationRoutine.Id, migrationRoutine.Name)); - saveConfiguration(migrationOptions); - logger.LogDebug("Migration '{Name}' marked as applied in configuration.", migrationRoutine.Name); - } - } - catch (System.Exception) when (migrationKey is not null && jellyfinDatabaseProvider is not null) - { - logger.LogInformation("Rollback on database as migration reported failure."); - await jellyfinDatabaseProvider.RestoreBackupFast(migrationKey, CancellationToken.None).ConfigureAwait(false); - throw; - } - } - } -} diff --git a/Jellyfin.Server/Migrations/MigrationsFactory.cs b/Jellyfin.Server/Migrations/MigrationsFactory.cs deleted file mode 100644 index 23c1b1ee6..000000000 --- a/Jellyfin.Server/Migrations/MigrationsFactory.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Collections.Generic; -using MediaBrowser.Common.Configuration; - -namespace Jellyfin.Server.Migrations -{ - /// <summary> - /// A factory that can find a persistent file of the migration configuration, which lists all applied migrations. - /// </summary> - public class MigrationsFactory : IConfigurationFactory - { - /// <inheritdoc/> - public IEnumerable<ConfigurationStore> GetConfigurations() - { - return new[] - { - new MigrationsListStore() - }; - } - } -} diff --git a/Jellyfin.Server/Migrations/MigrationsListStore.cs b/Jellyfin.Server/Migrations/MigrationsListStore.cs deleted file mode 100644 index 7a1ca6671..000000000 --- a/Jellyfin.Server/Migrations/MigrationsListStore.cs +++ /dev/null @@ -1,24 +0,0 @@ -using MediaBrowser.Common.Configuration; - -namespace Jellyfin.Server.Migrations -{ - /// <summary> - /// A configuration that lists all the migration routines that were applied. - /// </summary> - public class MigrationsListStore : ConfigurationStore - { - /// <summary> - /// The name of the configuration in the storage. - /// </summary> - public static readonly string StoreKey = "migrations"; - - /// <summary> - /// Initializes a new instance of the <see cref="MigrationsListStore"/> class. - /// </summary> - public MigrationsListStore() - { - ConfigurationType = typeof(MigrationOptions); - Key = StoreKey; - } - } -} diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/CreateNetworkConfiguration.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/CreateNetworkConfiguration.cs index 8462d0a8c..fd472cff7 100644 --- a/Jellyfin.Server/Migrations/PreStartupRoutines/CreateNetworkConfiguration.cs +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/CreateNetworkConfiguration.cs @@ -8,7 +8,10 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.PreStartupRoutines; /// <inheritdoc /> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T00:00:00", nameof(CreateNetworkConfiguration), "9B354818-94D5-4B68-AC49-E35CB85F9D84", Stage = Stages.JellyfinMigrationStageTypes.PreInitialisation)] public class CreateNetworkConfiguration : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ServerApplicationPaths _applicationPaths; private readonly ILogger<CreateNetworkConfiguration> _logger; @@ -25,15 +28,6 @@ public class CreateNetworkConfiguration : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => Guid.Parse("9B354818-94D5-4B68-AC49-E35CB85F9D84"); - - /// <inheritdoc /> - public string Name => nameof(CreateNetworkConfiguration); - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "network.xml"); diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateEncodingOptions.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateEncodingOptions.cs index 61f5620dc..0141b43c9 100644 --- a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateEncodingOptions.cs +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateEncodingOptions.cs @@ -10,7 +10,10 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.PreStartupRoutines; /// <inheritdoc /> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T03:00:00", nameof(MigrateEncodingOptions), "A8E61960-7726-4450-8F3D-82C12DAABBCB", Stage = Stages.JellyfinMigrationStageTypes.PreInitialisation)] public class MigrateEncodingOptions : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ServerApplicationPaths _applicationPaths; private readonly ILogger<MigrateEncodingOptions> _logger; @@ -27,15 +30,6 @@ public class MigrateEncodingOptions : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => Guid.Parse("A8E61960-7726-4450-8F3D-82C12DAABBCB"); - - /// <inheritdoc /> - public string Name => nameof(MigrateEncodingOptions); - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "encoding.xml"); diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateMusicBrainzTimeout.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateMusicBrainzTimeout.cs index 580282a5f..e8da9f515 100644 --- a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateMusicBrainzTimeout.cs +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateMusicBrainzTimeout.cs @@ -9,7 +9,10 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.PreStartupRoutines; /// <inheritdoc /> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T02:00:00", nameof(MigrateMusicBrainzTimeout), "A6DCACF4-C057-4Ef9-80D3-61CEF9DDB4F0", Stage = Stages.JellyfinMigrationStageTypes.PreInitialisation)] public class MigrateMusicBrainzTimeout : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ServerApplicationPaths _applicationPaths; private readonly ILogger<MigrateMusicBrainzTimeout> _logger; @@ -26,15 +29,6 @@ public class MigrateMusicBrainzTimeout : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => Guid.Parse("A6DCACF4-C057-4Ef9-80D3-61CEF9DDB4F0"); - - /// <inheritdoc /> - public string Name => nameof(MigrateMusicBrainzTimeout); - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { string path = Path.Combine(_applicationPaths.PluginConfigurationsPath, "Jellyfin.Plugin.MusicBrainz.xml"); diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateNetworkConfiguration.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateNetworkConfiguration.cs index 09b292171..f2790c1a1 100644 --- a/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateNetworkConfiguration.cs +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/MigrateNetworkConfiguration.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.PreStartupRoutines; /// <inheritdoc /> +[JellyfinMigration("2025-04-20T01:00:00", nameof(MigrateNetworkConfiguration), "4FB5C950-1991-11EE-9B4B-0800200C9A66", Stage = Stages.JellyfinMigrationStageTypes.PreInitialisation)] public class MigrateNetworkConfiguration : IMigrationRoutine { private readonly ServerApplicationPaths _applicationPaths; @@ -28,15 +29,6 @@ public class MigrateNetworkConfiguration : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => Guid.Parse("4FB5C950-1991-11EE-9B4B-0800200C9A66"); - - /// <inheritdoc /> - public string Name => nameof(MigrateNetworkConfiguration); - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "network.xml"); diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs index 0a37b35a6..995b2bbf9 100644 --- a/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs @@ -9,7 +9,10 @@ using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.PreStartupRoutines; /// <inheritdoc /> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T04:00:00", nameof(RenameEnableGroupingIntoCollections), "E73B777D-CD5C-4E71-957A-B86B3660B7CF", Stage = Stages.JellyfinMigrationStageTypes.PreInitialisation)] public class RenameEnableGroupingIntoCollections : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ServerApplicationPaths _applicationPaths; private readonly ILogger<RenameEnableGroupingIntoCollections> _logger; @@ -26,15 +29,6 @@ public class RenameEnableGroupingIntoCollections : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => Guid.Parse("E73B777D-CD5C-4E71-957A-B86B3660B7CF"); - - /// <inheritdoc /> - public string Name => nameof(RenameEnableGroupingIntoCollections); - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "system.xml"); diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs index 2047ec743..00d152b4b 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to add the default cast receivers to the system config. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T16:00:00", nameof(AddDefaultCastReceivers), "34A1A1C4-5572-418E-A2F8-32CDFE2668E8", RunMigrationOnSetup = true)] public class AddDefaultCastReceivers : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -21,15 +24,6 @@ public class AddDefaultCastReceivers : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("34A1A1C4-5572-418E-A2F8-32CDFE2668E8"); - - /// <inheritdoc /> - public string Name => "AddDefaultCastReceivers"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { _serverConfigurationManager.Configuration.CastReceiverApplications = diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs index fc6b5d597..8c8398a16 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Migration to initialize system configuration with the default plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T09:00:00", nameof(AddDefaultPluginRepository), "EB58EBEE-9514-4B9B-8225-12E1A40020DF", RunMigrationOnSetup = true)] public class AddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -27,15 +30,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("EB58EBEE-9514-4B9B-8225-12E1A40020DF"); - - /// <inheritdoc/> - public string Name => "AddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; - - /// <inheritdoc/> public void Perform() { _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; diff --git a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs index 5a8ef2e1c..1326a6dc8 100644 --- a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs +++ b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs @@ -12,7 +12,10 @@ namespace Jellyfin.Server.Migrations.Routines /// If the deprecated logging.json file exists and has a custom config, it will be used as logging.user.json, /// otherwise a blank file will be created. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T06:00:00", nameof(CreateUserLoggingConfigFile), "EF103419-8451-40D8-9F34-D1A8E93A1679")] internal class CreateUserLoggingConfigFile : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { /// <summary> /// File history for logging.json as existed during this migration creation. The contents for each has been minified. @@ -43,15 +46,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{EF103419-8451-40D8-9F34-D1A8E93A1679}"); - - /// <inheritdoc/> - public string Name => "CreateLoggingConfigHierarchy"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logDirectory = _appPaths.ConfigurationDirectoryPath; diff --git a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs index 378e88e25..acf2835fe 100644 --- a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs +++ b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Disable transcode throttling for all installations since it is currently broken for certain video formats. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T05:00:00", nameof(DisableTranscodingThrottling), "4124C2CD-E939-4FFB-9BE9-9B311C413638")] internal class DisableTranscodingThrottling : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<DisableTranscodingThrottling> _logger; private readonly IConfigurationManager _configManager; @@ -19,15 +22,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{4124C2CD-E939-4FFB-9BE9-9B311C413638}"); - - /// <inheritdoc/> - public string Name => "DisableTranscodingThrottling"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { // Set EnableThrottling to false since it wasn't used before and may introduce issues diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index a20253369..05ded06ba 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -16,9 +16,12 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Fixes the data column of audio types to be deserializable. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T18:00:00", nameof(FixAudioData), "CF6FABC2-9FBE-4933-84A5-FFE52EF22A58")] + [JellyfinMigrationBackup(LegacyLibraryDb = true)] internal class FixAudioData : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private const string DbFilename = "library.db"; private readonly ILogger<FixAudioData> _logger; private readonly IServerApplicationPaths _applicationPaths; private readonly IItemRepository _itemRepository; @@ -34,40 +37,8 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{CF6FABC2-9FBE-4933-84A5-FFE52EF22A58}"); - - /// <inheritdoc/> - public string Name => "FixAudioData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) - { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); - File.Copy(dbPath, bakPath); - _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } - } - _logger.LogInformation("Backfilling audio lyrics data to database."); var startIndex = 0; var records = _itemRepository.GetCount(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs index 192c170b2..56614ece3 100644 --- a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs +++ b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs @@ -13,7 +13,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Properly set playlist owner. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T15:00:00", nameof(FixPlaylistOwner), "615DFA9E-2497-4DBB-A472-61938B752C5B")] internal class FixPlaylistOwner : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<FixPlaylistOwner> _logger; private readonly ILibraryManager _libraryManager; @@ -30,15 +33,6 @@ internal class FixPlaylistOwner : IMigrationRoutine } /// <inheritdoc/> - public Guid Id => Guid.Parse("{615DFA9E-2497-4DBB-A472-61938B752C5B}"); - - /// <inheritdoc/> - public string Name => "FixPlaylistOwner"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs index e9fe9abce..a954d307e 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs @@ -14,7 +14,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the activity log database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T07:00:00", nameof(MigrateActivityLogDb), "3793eb59-bc8c-456c-8b9f-bd5a62a42978")] public class MigrateActivityLogDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "activitylog.db"; @@ -36,15 +39,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("3793eb59-bc8c-456c-8b9f-bd5a62a42978"); - - /// <inheritdoc/> - public string Name => "MigrateActivityLogDatabase"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logLevelDictionary = new Dictionary<string, LogLevel>(StringComparer.OrdinalIgnoreCase) diff --git a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs index feaf46c84..c6699c21d 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs @@ -15,7 +15,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// A migration that moves data from the authentication database into the new schema. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T14:00:00", nameof(MigrateAuthenticationDb), "5BD72F41-E6F3-4F60-90AA-09869ABE0E22")] public class MigrateAuthenticationDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "authentication.db"; @@ -44,15 +47,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("5BD72F41-E6F3-4F60-90AA-09869ABE0E22"); - - /// <inheritdoc /> - public string Name => "MigrateAuthenticationDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { var dataPath = _appPaths.DataPath; diff --git a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs index a8fa2e52a..0d9952ce9 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs @@ -20,7 +20,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the display preferences database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T12:00:00", nameof(MigrateDisplayPreferencesDb), "06387815-C3CC-421F-A888-FB5F9992BEA8")] public class MigrateDisplayPreferencesDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "displaypreferences.db"; @@ -52,15 +55,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("06387815-C3CC-421F-A888-FB5F9992BEA8"); - - /// <inheritdoc /> - public string Name => "MigrateDisplayPreferencesDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { HomeSectionType[] defaults = diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs new file mode 100644 index 000000000..033045e63 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -0,0 +1,152 @@ +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions.Json; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-21T00:00:00", nameof(MigrateKeyframeData))] +public class MigrateKeyframeData : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IApplicationPaths _appPaths; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="dbProvider">The EFCore db factory.</param> + public MigrateKeyframeData( + IStartupLogger startupLogger, + IApplicationPaths appPaths, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _logger = startupLogger; + _appPaths = appPaths; + _dbProvider = dbProvider; + } + + private string KeyframeCachePath => Path.Combine(_appPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var baseQuery = context.BaseItems.Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder).OrderBy(e => e.Id); + var records = baseQuery.Count(); + _logger.LogInformation("Checking {Count} items for importable keyframe data.", records); + + context.KeyframeData.ExecuteDelete(); + using var transaction = context.Database.BeginTransaction(); + do + { + var results = baseQuery.Skip(offset).Take(Limit).Select(b => new Tuple<Guid, string?>(b.Id, b.Path)).ToList(); + foreach (var result in results) + { + if (TryGetKeyframeData(result.Item1, result.Item2, out var data)) + { + itemCount++; + context.KeyframeData.Add(data); + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Imported: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + transaction.Commit(); + + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + + if (Directory.Exists(KeyframeCachePath)) + { + Directory.Delete(KeyframeCachePath, true); + } + } + + private bool TryGetKeyframeData(Guid id, string? path, [NotNullWhen(true)] out KeyframeData? data) + { + data = null; + if (!string.IsNullOrEmpty(path)) + { + var cachePath = GetCachePath(KeyframeCachePath, path); + if (TryReadFromCache(cachePath, out var keyframeData)) + { + data = new() + { + ItemId = id, + KeyframeTicks = keyframeData.KeyframeTicks.ToList(), + TotalDuration = keyframeData.TotalDuration + }; + + return true; + } + } + + return false; + } + + private string? GetCachePath(string keyframeCachePath, string filePath) + { + DateTime? lastWriteTimeUtc; + try + { + lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping {Path}: {Exception}", filePath, e.Message); + + return null; + } + + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string? cachePath, [NotNullWhen(true)] out MediaEncoding.Keyframes.KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<MediaEncoding.Keyframes.KeyframeData>(bytes, _jsonOptions); + + return cachedResult is not null; + } + + cachedResult = null; + + return false; + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index 427f04f9d..521655a4f 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -9,12 +9,12 @@ using System.Globalization; using System.IO; using System.Linq; using System.Text; -using System.Threading; using Emby.Server.Implementations.Data; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; @@ -29,11 +29,13 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// The migration routine for migrating the userdata database to EF Core. /// </summary> +[JellyfinMigration("2025-04-20T20:00:00", nameof(MigrateLibraryDb))] +[JellyfinMigrationBackup(JellyfinDb = true, LegacyLibraryDb = true)] internal class MigrateLibraryDb : IDatabaseMigrationRoutine { private const string DbFilename = "library.db"; - private readonly ILogger<MigrateLibraryDb> _logger; + private readonly IStartupLogger _logger; private readonly IServerApplicationPaths _paths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; private readonly IDbContextFactory<JellyfinDbContext> _provider; @@ -41,305 +43,379 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine /// <summary> /// Initializes a new instance of the <see cref="MigrateLibraryDb"/> class. /// </summary> - /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="provider">The database provider.</param> /// <param name="paths">The server application paths.</param> /// <param name="jellyfinDatabaseProvider">The database provider for special access.</param> public MigrateLibraryDb( - ILogger<MigrateLibraryDb> logger, + IStartupLogger startupLogger, IDbContextFactory<JellyfinDbContext> provider, IServerApplicationPaths paths, IJellyfinDatabaseProvider jellyfinDatabaseProvider) { - _logger = logger; + _logger = startupLogger; _provider = provider; _paths = paths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; } /// <inheritdoc/> - public Guid Id => Guid.Parse("36445464-849f-429f-9ad0-bb130efa0664"); - - /// <inheritdoc/> - public string Name => "MigrateLibraryDbData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; // TODO Change back after testing - - /// <inheritdoc/> public void Perform() { _logger.LogInformation("Migrating the userdata from library.db may take a while, do not stop Jellyfin."); var dataPath = _paths.DataPath; var libraryDbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={libraryDbPath}"); - var migrationTotalTime = TimeSpan.Zero; + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } - var stopwatch = new Stopwatch(); - stopwatch.Start(); + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving TypedBaseItem."); - const string typedBaseItemsQuery = """ - SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, - IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, - PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, - ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, - Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, - DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, - PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, - ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems - """; - dbContext.BaseItems.ExecuteDelete(); + var fullOperationTimer = new Stopwatch(); + fullOperationTimer.Start(); - var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); - foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + using (var operation = GetPreparedDbContext("Cleanup database")) { - var baseItem = GetItem(dto); - dbContext.BaseItems.Add(baseItem.BaseItem); - foreach (var dataKey in baseItem.LegacyUserDataKey) - { - legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; - } + operation.JellyfinDbContext.AttachmentStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.BaseItems.ExecuteDelete(); + operation.JellyfinDbContext.ItemValues.ExecuteDelete(); + operation.JellyfinDbContext.UserData.ExecuteDelete(); + operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.Peoples.ExecuteDelete(); + operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete(); + operation.JellyfinDbContext.Chapters.ExecuteDelete(); + operation.JellyfinDbContext.AncestorIds.ExecuteDelete(); } - _logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); + connection.Open(); - _logger.LogInformation("Start moving ItemValues."); - // do not migrate inherited types as they are now properly mapped in search and lookup. - const string itemValueQuery = - """ - SELECT ItemId, Type, Value, CleanValue FROM ItemValues - WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) - """; - dbContext.ItemValues.ExecuteDelete(); + var baseItemIds = new HashSet<Guid>(); + using (var operation = GetPreparedDbContext("moving TypedBaseItem")) + { + const string typedBaseItemsQuery = + """ + SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, + IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, + PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, + ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, + Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, + DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, + PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, + ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems + """; + using (new TrackedMigrationStep("Loading TypedBaseItems", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + { + var baseItem = GetItem(dto); + operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem); + baseItemIds.Add(baseItem.BaseItem.Id); + foreach (var dataKey in baseItem.LegacyUserDataKey) + { + legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + } + } + } - // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. - var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } - foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + using (var operation = GetPreparedDbContext("moving ItemValues")) { - var itemId = dto.GetGuid(0); - var entity = GetItemValue(dto); - var key = ((int)entity.Type, entity.CleanValue); - if (!localItems.TryGetValue(key, out var existing)) + // do not migrate inherited types as they are now properly mapped in search and lookup. + const string itemValueQuery = + """ + SELECT ItemId, Type, Value, CleanValue FROM ItemValues + WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) + """; + + // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. + var localItems = new Dictionary<(int Type, string Value), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep("loading ItemValues", _logger)) { - localItems[key] = existing = (entity, []); + foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + { + var itemId = dto.GetGuid(0); + var entity = GetItemValue(dto); + var key = ((int)entity.Type, entity.Value); + if (!localItems.TryGetValue(key, out var existing)) + { + localItems[key] = existing = (entity, []); + } + + existing.ItemIds.Add(itemId); + } + + foreach (var item in localItems) + { + operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue); + operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + { + Item = null!, + ItemValue = null!, + ItemId = f, + ItemValueId = item.Value.ItemValue.ItemValueId + })); + } } - existing.ItemIds.Add(itemId); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - foreach (var item in localItems) + using (var operation = GetPreparedDbContext("moving UserData")) { - dbContext.ItemValues.Add(item.Value.ItemValue); - dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + var queryResult = connection.Query( + """ + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) + """); + + using (new TrackedMigrationStep("loading UserData", _logger)) { - Item = null!, - ItemValue = null!, - ItemId = f, - ItemValueId = item.Value.ItemValue.ItemValueId - })); - } + var users = operation.JellyfinDbContext.Users.AsNoTracking().ToImmutableArray(); + var userIdBlacklist = new HashSet<int>(); - _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + foreach (var entity in queryResult) + { + var userData = GetUserData(users, entity, userIdBlacklist); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); - _logger.LogInformation("Start moving UserData."); - var queryResult = connection.Query(""" - SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) - """); + continue; + } - dbContext.UserData.ExecuteDelete(); + if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + { + _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); + continue; + } - var users = dbContext.Users.AsNoTracking().ToImmutableArray(); + userData.ItemId = refItem.Id; + operation.JellyfinDbContext.UserData.Add(userData); + } - foreach (var entity in queryResult) - { - var userData = GetUserData(users, entity); - if (userData is null) - { - _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0)); - continue; + users.Clear(); } - if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + legacyBaseItemWithUserKeys.Clear(); + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger)) { - _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); - continue; + operation.JellyfinDbContext.SaveChanges(); } - - userData.ItemId = refItem.Id; - dbContext.UserData.Add(userData); } - users.Clear(); - legacyBaseItemWithUserKeys.Clear(); - _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count); - dbContext.SaveChanges(); - - _logger.LogInformation("Start moving MediaStreamInfos."); - const string mediaStreamQuery = """ - SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, - IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, - AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, - Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, - DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired - FROM MediaStreams - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) - """; - dbContext.MediaStreamInfos.ExecuteDelete(); - - foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + using (var operation = GetPreparedDbContext("moving MediaStreamInfos")) { - dbContext.MediaStreamInfos.Add(GetMediaStream(dto)); - } + const string mediaStreamQuery = + """ + SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, + IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, + AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, + Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, + DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired + FROM MediaStreams + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) + """; - _logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving People."); - const string personsQuery = """ - SELECT ItemId, Name, Role, PersonType, SortOrder FROM People - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) - """; - dbContext.Peoples.ExecuteDelete(); - dbContext.PeopleBaseItemMap.ExecuteDelete(); - - var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - var baseItemIds = dbContext.BaseItems.Select(b => b.Id).ToHashSet(); + using (new TrackedMigrationStep("loading MediaStreamInfos", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + { + operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto)); + } + } - foreach (SqliteDataReader reader in connection.Query(personsQuery)) - { - var itemId = reader.GetGuid(0); - if (!baseItemIds.Contains(itemId)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger)) { - _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); - continue; + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving AttachmentStreamInfos")) + { + const string mediaAttachmentQuery = + """ + SELECT ItemId, AttachmentIndex, Codec, CodecTag, Comment, filename, MIMEType + FROM mediaattachments + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = mediaattachments.ItemId) + """; - var entity = GetPerson(reader); - if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + using (new TrackedMigrationStep("loading AttachmentStreamInfos", _logger)) { - peopleCache[entity.Name] = personCache = (entity, []); + foreach (SqliteDataReader dto in connection.Query(mediaAttachmentQuery)) + { + operation.JellyfinDbContext.AttachmentStreamInfos.Add(GetMediaAttachment(dto)); + } } - if (reader.TryGetString(2, out var role)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AttachmentStreamInfos.Local.Count} AttachmentStreamInfos entries", _logger)) { + operation.JellyfinDbContext.SaveChanges(); } + } - int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); + using (var operation = GetPreparedDbContext("moving People")) + { + const string personsQuery = + """ + SELECT ItemId, Name, Role, PersonType, SortOrder FROM People + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) + """; - personCache.Items.Add(new PeopleBaseItemMap() + var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); + + using (new TrackedMigrationStep("loading People", _logger)) { - Item = null!, - ItemId = itemId, - People = null!, - PeopleId = personCache.Person.Id, - ListOrder = sortOrder, - SortOrder = sortOrder, - Role = role - }); - } + foreach (SqliteDataReader reader in connection.Query(personsQuery)) + { + var itemId = reader.GetGuid(0); + if (!baseItemIds.Contains(itemId)) + { + _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); + continue; + } - baseItemIds.Clear(); + var entity = GetPerson(reader); + if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + { + peopleCache[entity.Name] = personCache = (entity, []); + } - foreach (var item in peopleCache) - { - dbContext.Peoples.Add(item.Value.Person); - dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); - } + if (reader.TryGetString(2, out var role)) + { + } - peopleCache.Clear(); + int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); - _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + personCache.Items.Add(new PeopleBaseItemMap() + { + Item = null!, + ItemId = itemId, + People = null!, + PeopleId = personCache.Person.Id, + ListOrder = sortOrder, + SortOrder = sortOrder, + Role = role + }); + } - _logger.LogInformation("Start moving Chapters."); - const string chapterQuery = """ - SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) - """; - dbContext.Chapters.ExecuteDelete(); + baseItemIds.Clear(); - foreach (SqliteDataReader dto in connection.Query(chapterQuery)) - { - var chapter = GetChapter(dto); - dbContext.Chapters.Add(chapter); - } + foreach (var item in peopleCache) + { + operation.JellyfinDbContext.Peoples.Add(item.Value.Person); + operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); + } - _logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + peopleCache.Clear(); + } - _logger.LogInformation("Start moving AncestorIds."); - const string ancestorIdsQuery = """ - SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds - WHERE - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) - AND - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) - """; - dbContext.Chapters.ExecuteDelete(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } - foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + using (var operation = GetPreparedDbContext("moving Chapters")) { - var ancestorId = GetAncestorId(dto); - dbContext.AncestorIds.Add(ancestorId); + const string chapterQuery = + """ + SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) + """; + + using (new TrackedMigrationStep("loading Chapters", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(chapterQuery)) + { + var chapter = GetChapter(dto); + operation.JellyfinDbContext.Chapters.Add(chapter); + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.Chapters.Local.Count); + using (var operation = GetPreparedDbContext("moving AncestorIds")) + { + const string ancestorIdsQuery = + """ + SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds + WHERE + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) + AND + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) + """; - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (new TrackedMigrationStep("loading AncestorIds", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + { + var ancestorId = GetAncestorId(dto); + operation.JellyfinDbContext.AncestorIds.Add(ancestorId); + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } connection.Close(); + _logger.LogInformation("Migration of the Library.db done."); - _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + _logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed); SqliteConnection.ClearAllPools(); + _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); File.Move(libraryDbPath, libraryDbPath + ".old", true); + } - _logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime); - - _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); + private DatabaseMigrationStep GetPreparedDbContext(string operationName) + { + var dbContext = _provider.CreateDbContext(); + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return new DatabaseMigrationStep(dbContext, operationName, _logger); } - private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto) + private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto, HashSet<int> userIdBlacklist) { var internalUserId = dto.GetInt32(1); var user = users.FirstOrDefault(e => e.InternalId == internalUserId); if (user is null) { + if (userIdBlacklist.Contains(internalUserId)) + { + return null; + } + _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); return null; } @@ -654,6 +730,48 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return item; } + /// <summary> + /// Gets the attachment. + /// </summary> + /// <param name="reader">The reader.</param> + /// <returns>MediaAttachment.</returns> + private AttachmentStreamInfo GetMediaAttachment(SqliteDataReader reader) + { + var item = new AttachmentStreamInfo + { + Index = reader.GetInt32(1), + Item = null!, + ItemId = reader.GetGuid(0), + }; + + if (reader.TryGetString(2, out var codec)) + { + item.Codec = codec; + } + + if (reader.TryGetString(3, out var codecTag)) + { + item.CodecTag = codecTag; + } + + if (reader.TryGetString(4, out var comment)) + { + item.Comment = comment; + } + + if (reader.TryGetString(5, out var fileName)) + { + item.Filename = fileName; + } + + if (reader.TryGetString(6, out var mimeType)) + { + item.MimeType = mimeType; + } + + return item; + } + private (BaseItemEntity BaseItem, string[] LegacyUserDataKey) GetItem(SqliteDataReader reader) { var entity = new BaseItemEntity() @@ -1214,4 +1332,58 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return image; } + + private class TrackedMigrationStep : IDisposable + { + private readonly string _operationName; + private readonly ILogger _logger; + private readonly Stopwatch _operationTimer; + private bool _disposed; + + public TrackedMigrationStep(string operationName, ILogger logger) + { + _operationName = operationName; + _logger = logger; + _operationTimer = Stopwatch.StartNew(); + logger.LogInformation("Start {OperationName}", operationName); + } + + public bool Disposed + { + get => _disposed; + set => _disposed = value; + } + + public virtual void Dispose() + { + if (Disposed) + { + return; + } + + Disposed = true; + _logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed); + } + } + + private sealed class DatabaseMigrationStep : TrackedMigrationStep + { + public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger) + { + JellyfinDbContext = jellyfinDbContext; + } + + public JellyfinDbContext JellyfinDbContext { get; } + + public override void Dispose() + { + if (Disposed) + { + return; + } + + JellyfinDbContext.Dispose(); + base.Dispose(); + } + } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs new file mode 100644 index 000000000..2d5fc2a0d --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs @@ -0,0 +1,73 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for checking if the current instance of Jellyfin is compatiable to be upgraded. +/// </summary> +[JellyfinMigration("2025-04-20T19:30:00", nameof(MigrateLibraryDbCompatibilityCheck))] +public class MigrateLibraryDbCompatibilityCheck : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db"; + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateLibraryDbCompatibilityCheck"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger.</param> + /// <param name="paths">The Path service.</param> + public MigrateLibraryDbCompatibilityCheck(IStartupLogger startupLogger, IServerApplicationPaths paths) + { + _logger = startupLogger; + _paths = paths; + } + + /// <inheritdoc/> + public async Task PerformAsync(CancellationToken cancellationToken) + { + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } + + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + CheckMigratableVersion(connection); + await connection.CloseAsync().ConfigureAwait(false); + } + + private static void CheckMigratableVersion(SqliteConnection connection) + { + CheckColumnExistance(connection, "TypedBaseItems", "lufs"); + CheckColumnExistance(connection, "TypedBaseItems", "normalizationgain"); + CheckColumnExistance(connection, "mediastreams", "dvversionmajor"); + + static void CheckColumnExistance(SqliteConnection connection, string table, string column) + { + using (var cmd = connection.CreateCommand()) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + cmd.CommandText = $"Select COUNT(1) FROM pragma_table_xinfo('{table}') WHERE lower(name) = '{column}';"; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + var result = cmd.ExecuteScalar()!; + if (!result.Equals(1L)) + { + throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or above are supported. Please upgrade first to server version 10.10.7 before attempting to upgrade afterwards to 10.11"); + } + } + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 9c2184029..ae93557de 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,94 +1,69 @@ using System; -using System.Globalization; -using System.IO; -using Emby.Server.Implementations.Data; -using MediaBrowser.Controller; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Model.Globalization; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Migrate rating levels to new rating level system. - /// </summary> - internal class MigrateRatingLevels : IMigrationRoutine - { - private const string DbFilename = "library.db"; - private readonly ILogger<MigrateRatingLevels> _logger; - private readonly IServerApplicationPaths _applicationPaths; - private readonly ILocalizationManager _localizationManager; - - public MigrateRatingLevels( - IServerApplicationPaths applicationPaths, - ILoggerFactory loggerFactory, - ILocalizationManager localizationManager) - { - _applicationPaths = applicationPaths; - _localizationManager = localizationManager; - _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{73DAB92A-178B-48CD-B05B-FE18733ACDC8}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "MigrateRatingLevels"; +/// <summary> +/// Migrate rating levels. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T22:00:00", nameof(MigrateRatingLevels))] +[JellyfinMigrationBackup(JellyfinDb = true)] +#pragma warning restore CS0618 // Type or member is obsolete +internal class MigrateRatingLevels : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly ILocalizationManager _localizationManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public MigrateRatingLevels( + IDbContextFactory<JellyfinDbContext> provider, + IStartupLogger logger, + ILocalizationManager localizationManager) + { + _provider = provider; + _localizationManager = localizationManager; + _logger = logger; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + _logger.LogInformation("Recalculating parental rating levels based on rating string."); + using var context = _provider.CreateDbContext(); + using var transaction = context.Database.BeginTransaction(); + var ratings = context.BaseItems.AsNoTracking().Select(e => e.OfficialRating).Distinct(); + foreach (var rating in ratings) { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) + if (string.IsNullOrEmpty(rating)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } + int? value = null; + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, value)); + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, value)); } - - // Migrate parental rating strings to new levels - _logger.LogInformation("Recalculating parental rating levels based on rating string."); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + else { - var queryResult = connection.Query("SELECT DISTINCT OfficialRating FROM TypedBaseItems"); - foreach (var entry in queryResult) - { - if (!entry.TryGetString(0, out var ratingString) || string.IsNullOrEmpty(ratingString)) - { - connection.Execute("UPDATE TypedBaseItems SET InheritedParentalRatingValue = NULL WHERE OfficialRating IS NULL OR OfficialRating='';"); - } - else - { - var ratingValue = _localizationManager.GetRatingLevel(ratingString)?.ToString(CultureInfo.InvariantCulture) ?? "NULL"; - - using var statement = connection.PrepareStatement("UPDATE TypedBaseItems SET InheritedParentalRatingValue = @Value WHERE OfficialRating = @Rating;"); - statement.TryBind("@Value", ratingValue); - statement.TryBind("@Rating", ratingString); - statement.ExecuteNonQuery(); - } - } - - transaction.Commit(); + var ratingValue = _localizationManager.GetRatingScore(rating); + var score = ratingValue?.Score; + var subScore = ratingValue?.SubScore; + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, score)); + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, subScore)); } } + + transaction.Commit(); } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs index c40560660..e5584fb94 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs @@ -17,206 +17,200 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using JsonSerializer = System.Text.Json.JsonSerializer; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for migrating the user database to EF Core. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T10:00:00", nameof(MigrateUserDb), "5C4B82A2-F053-4009-BD05-B6FCAD82F14C")] +public class MigrateUserDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private const string DbFilename = "users.db"; + + private readonly ILogger<MigrateUserDb> _logger; + private readonly IServerApplicationPaths _paths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly IXmlSerializer _xmlSerializer; + /// <summary> - /// The migration routine for migrating the user database to EF Core. + /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. /// </summary> - public class MigrateUserDb : IMigrationRoutine + /// <param name="logger">The logger.</param> + /// <param name="paths">The server application paths.</param> + /// <param name="provider">The database provider.</param> + /// <param name="xmlSerializer">The xml serializer.</param> + public MigrateUserDb( + ILogger<MigrateUserDb> logger, + IServerApplicationPaths paths, + IDbContextFactory<JellyfinDbContext> provider, + IXmlSerializer xmlSerializer) { - private const string DbFilename = "users.db"; - - private readonly ILogger<MigrateUserDb> _logger; - private readonly IServerApplicationPaths _paths; - private readonly IDbContextFactory<JellyfinDbContext> _provider; - private readonly IXmlSerializer _xmlSerializer; - - /// <summary> - /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. - /// </summary> - /// <param name="logger">The logger.</param> - /// <param name="paths">The server application paths.</param> - /// <param name="provider">The database provider.</param> - /// <param name="xmlSerializer">The xml serializer.</param> - public MigrateUserDb( - ILogger<MigrateUserDb> logger, - IServerApplicationPaths paths, - IDbContextFactory<JellyfinDbContext> provider, - IXmlSerializer xmlSerializer) - { - _logger = logger; - _paths = paths; - _provider = provider; - _xmlSerializer = xmlSerializer; - } + _logger = logger; + _paths = paths; + _provider = provider; + _xmlSerializer = xmlSerializer; + } - /// <inheritdoc/> - public Guid Id => Guid.Parse("5C4B82A2-F053-4009-BD05-B6FCAD82F14C"); + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); - /// <inheritdoc/> - public string Name => "MigrateUserDatabase"; + using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + { + connection.Open(); + using var dbContext = _provider.CreateDbContext(); - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); + dbContext.RemoveRange(dbContext.Users); + dbContext.SaveChanges(); - using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + foreach (var entry in queryResult) { - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); + UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); + if (mockup is null) + { + continue; + } - dbContext.RemoveRange(dbContext.Users); - dbContext.SaveChanges(); + var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); + + var configPath = Path.Combine(userDataDir, "config.xml"); + var config = File.Exists(configPath) + ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() + : new UserConfiguration(); + + var policyPath = Path.Combine(userDataDir, "policy.xml"); + var policy = File.Exists(policyPath) + ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() + : new UserPolicy(); + policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( + "Emby.Server.Implementations.Library", + "Jellyfin.Server.Implementations.Users", + StringComparison.Ordinal) + ?? typeof(DefaultAuthenticationProvider).FullName; + + policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; + int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch + { + -1 => null, + 0 => 3, + _ => policy.LoginAttemptsBeforeLockout + }; - foreach (var entry in queryResult) + var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) { - UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); - if (mockup is null) - { - continue; - } - - var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); - - var configPath = Path.Combine(userDataDir, "config.xml"); - var config = File.Exists(configPath) - ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() - : new UserConfiguration(); - - var policyPath = Path.Combine(userDataDir, "policy.xml"); - var policy = File.Exists(policyPath) - ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() - : new UserPolicy(); - policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( - "Emby.Server.Implementations.Library", - "Jellyfin.Server.Implementations.Users", - StringComparison.Ordinal) - ?? typeof(DefaultAuthenticationProvider).FullName; - - policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; - int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch - { - -1 => null, - 0 => 3, - _ => policy.LoginAttemptsBeforeLockout - }; + Id = entry.GetGuid(1), + InternalId = entry.GetInt64(0), + MaxParentalRatingScore = policy.MaxParentalRating, + MaxParentalRatingSubScore = null, + EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, + RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, + InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, + LoginAttemptsBeforeLockout = maxLoginAttempts, + SubtitleMode = config.SubtitleMode, + HidePlayedInLatest = config.HidePlayedInLatest, + EnableLocalPassword = config.EnableLocalPassword, + PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, + DisplayCollectionsView = config.DisplayCollectionsView, + DisplayMissingEpisodes = config.DisplayMissingEpisodes, + AudioLanguagePreference = config.AudioLanguagePreference, + RememberAudioSelections = config.RememberAudioSelections, + EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, + RememberSubtitleSelections = config.RememberSubtitleSelections, + SubtitleLanguagePreference = config.SubtitleLanguagePreference, + Password = mockup.Password, + LastLoginDate = mockup.LastLoginDate, + LastActivityDate = mockup.LastActivityDate + }; + + if (mockup.ImageInfos.Length > 0) + { + ItemImageInfo info = mockup.ImageInfos[0]; - var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) + user.ProfileImage = new ImageInfo(info.Path) { - Id = entry.GetGuid(1), - InternalId = entry.GetInt64(0), - MaxParentalAgeRating = policy.MaxParentalRating, - EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, - RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, - InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, - LoginAttemptsBeforeLockout = maxLoginAttempts, - SubtitleMode = config.SubtitleMode, - HidePlayedInLatest = config.HidePlayedInLatest, - EnableLocalPassword = config.EnableLocalPassword, - PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, - DisplayCollectionsView = config.DisplayCollectionsView, - DisplayMissingEpisodes = config.DisplayMissingEpisodes, - AudioLanguagePreference = config.AudioLanguagePreference, - RememberAudioSelections = config.RememberAudioSelections, - EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, - RememberSubtitleSelections = config.RememberSubtitleSelections, - SubtitleLanguagePreference = config.SubtitleLanguagePreference, - Password = mockup.Password, - LastLoginDate = mockup.LastLoginDate, - LastActivityDate = mockup.LastActivityDate + LastModified = info.DateModified }; + } - if (mockup.ImageInfos.Length > 0) - { - ItemImageInfo info = mockup.ImageInfos[0]; - - user.ProfileImage = new ImageInfo(info.Path) - { - LastModified = info.DateModified - }; - } - - user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); - user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); - user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); - user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); - user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); - user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); - user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); - user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); - user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); - user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); - user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); - user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); - user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); - user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); - user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); - user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); - user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); - user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); - user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); - user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); - - foreach (var policyAccessSchedule in policy.AccessSchedules) - { - user.AccessSchedules.Add(policyAccessSchedule); - } - - user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); - user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); - user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); - user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); - user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); - user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); - user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); - user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); - user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); - - dbContext.Users.Add(user); + user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); + user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); + user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); + user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); + user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); + user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); + user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); + user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); + user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); + user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); + user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); + user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); + user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); + user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); + user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); + user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); + user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); + user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); + user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); + user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); + + foreach (var policyAccessSchedule in policy.AccessSchedules) + { + user.AccessSchedules.Add(policyAccessSchedule); } - dbContext.SaveChanges(); + user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); + user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); + user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); + user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); + user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); + user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); + user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); + user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); + user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); + + dbContext.Users.Add(user); } - try - { - File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + dbContext.SaveChanges(); + } - var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); - if (File.Exists(journalPath)) - { - File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); - } - } - catch (IOException e) + try + { + File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + + var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); + if (File.Exists(journalPath)) { - _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); } } + catch (IOException e) + { + _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + } + } #nullable disable - internal class UserMockup - { - public string Password { get; set; } + internal class UserMockup + { + public string Password { get; set; } - public string EasyPassword { get; set; } + public string EasyPassword { get; set; } - public DateTime? LastLoginDate { get; set; } + public DateTime? LastLoginDate { get; set; } - public DateTime? LastActivityDate { get; set; } + public DateTime? LastActivityDate { get; set; } - public string Name { get; set; } + public string Name { get; set; } - public ItemImageInfo[] ImageInfos { get; set; } - } + public ItemImageInfo[] ImageInfos { get; set; } } } diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs new file mode 100644 index 000000000..6f650f731 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -0,0 +1,295 @@ +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-20T21:00:00", nameof(MoveExtractedFiles))] +public class MoveExtractedFiles : IAsyncMigrationRoutine +{ + private readonly IApplicationPaths _appPaths; + private readonly ILogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IPathManager _pathManager; + private readonly IFileSystem _fileSystem; + + /// <summary> + /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class. + /// </summary> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + public MoveExtractedFiles( + IApplicationPaths appPaths, + ILogger<MoveExtractedFiles> logger, + IStartupLogger startupLogger, + IPathManager pathManager, + IFileSystem fileSystem, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _appPaths = appPaths; + _logger = startupLogger.With(logger); + _pathManager = pathManager; + _fileSystem = fileSystem; + _dbProvider = dbProvider; + } + + private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles"); + + private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); + + /// <inheritdoc /> + public async Task PerformAsync(CancellationToken cancellationToken) + { + const int Limit = 5000; + int itemCount = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder); + _logger.LogInformation("Checking {Count} items for movable extracted files.", records); + + // Make sure directories exist + Directory.CreateDirectory(SubtitleCachePath); + Directory.CreateDirectory(AttachmentCachePath); + + await foreach (var result in context.BaseItems + .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal)) + .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder) + .Select(b => new + { + b.Id, + b.Path, + b.MediaStreams + }) + .OrderBy(e => e.Id) + .WithPartitionProgress((partition) => _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", partition * Limit, itemCount, sw.Elapsed)) + .PartitionEagerAsync(Limit, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + if (MoveSubtitleAndAttachmentFiles(result.Id, result.Path, result.MediaStreams, context)) + { + itemCount++; + } + } + + _logger.LogInformation("Moved files for {Count} items in {Time}", itemCount, sw.Elapsed); + + // Get all subdirectories with 1 character names (those are the legacy directories) + var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList(); + subdirectories.AddRange(Directory.GetDirectories(AttachmentCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == AttachmentCachePath.Length + 2)); + + // Remove all legacy subdirectories + foreach (var subdir in subdirectories) + { + Directory.Delete(subdir, true); + } + + // Remove old cache path + var attachmentCachePath = Path.Join(_appPaths.CachePath, "attachments"); + if (Directory.Exists(attachmentCachePath)) + { + Directory.Delete(attachmentCachePath, true); + } + + _logger.LogInformation("Cleaned up left over subtitles and attachments."); + } + + private bool MoveSubtitleAndAttachmentFiles(Guid id, string? path, ICollection<MediaStreamInfo>? mediaStreams, JellyfinDbContext context) + { + var itemIdString = id.ToString("N", CultureInfo.InvariantCulture); + var modified = false; + if (mediaStreams is not null) + { + foreach (var mediaStream in mediaStreams) + { + if (mediaStream.Codec is null) + { + continue; + } + + var mediaStreamIndex = mediaStream.StreamIndex; + var extension = GetSubtitleExtension(mediaStream.Codec); + var oldSubtitleCachePath = GetOldSubtitleCachePath(path, mediaStreamIndex, extension); + if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath)) + { + continue; + } + + var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension); + if (File.Exists(newSubtitleCachePath)) + { + File.Delete(oldSubtitleCachePath); + } + else + { + var newDirectory = Path.GetDirectoryName(newSubtitleCachePath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldSubtitleCachePath, newSubtitleCachePath, false); + _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, id, oldSubtitleCachePath, newSubtitleCachePath); + + modified = true; + } + } + } + } + +#pragma warning disable CA1309 // Use ordinal string comparison + var attachments = context.AttachmentStreamInfos.Where(a => a.ItemId.Equals(id) && !string.Equals(a.Codec, "mjpeg")).ToList(); +#pragma warning restore CA1309 // Use ordinal string comparison + var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.Filename) + && (a.Filename.Contains('/', StringComparison.OrdinalIgnoreCase) || a.Filename.Contains('\\', StringComparison.OrdinalIgnoreCase))); + foreach (var attachment in attachments) + { + var attachmentIndex = attachment.Index; + var oldAttachmentPath = GetOldAttachmentDataPath(path, attachmentIndex); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + continue; + } + } + + var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.Filename ?? attachmentIndex.ToString(CultureInfo.InvariantCulture)); + if (File.Exists(newAttachmentPath)) + { + File.Delete(oldAttachmentPath); + } + else + { + var newDirectory = Path.GetDirectoryName(newAttachmentPath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldAttachmentPath, newAttachmentPath, false); + _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, id, oldAttachmentPath, newAttachmentPath); + + modified = true; + } + } + } + + return modified; + } + + private string? GetOldAttachmentDataPath(string? mediaPath, int attachmentStreamIndex) + { + if (mediaPath is null) + { + return null; + } + + string filename; + if (_fileSystem.IsPathFile(mediaPath)) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(mediaPath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping attachment at index {Index} for {Path}: {Exception}", attachmentStreamIndex, mediaPath, e.Message); + + return null; + } + + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + else + { + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + + return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename); + } + + private string? GetOldAttachmentCachePath(string mediaSourceId, AttachmentStreamInfo attachment, bool shouldExtractOneByOne) + { + var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId); + if (shouldExtractOneByOne) + { + return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture)); + } + + if (string.IsNullOrEmpty(attachment.Filename)) + { + return null; + } + + return Path.Join(attachmentFolderPath, attachment.Filename); + } + + private string? GetOldSubtitleCachePath(string? path, int streamIndex, string outputSubtitleExtension) + { + if (path is null) + { + return null; + } + + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(path); + } + catch (IOException e) + { + _logger.LogDebug("Skipping subtitle at index {Index} for {Path}: {Exception}", streamIndex, path, e.Message); + + return null; + } + + var ticksParam = string.Empty; + ReadOnlySpan<char> filename = new Guid(MD5.HashData(Encoding.Unicode.GetBytes(path + "_" + streamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture) + ticksParam))) + outputSubtitleExtension; + + return Path.Join(SubtitleCachePath, filename[..1], filename); + } + + private static string GetSubtitleExtension(string codec) + { + if (codec.ToLower(CultureInfo.InvariantCulture).Equals("ass", StringComparison.OrdinalIgnoreCase) + || codec.ToLower(CultureInfo.InvariantCulture).Equals("ssa", StringComparison.OrdinalIgnoreCase)) + { + return "." + codec; + } + else if (codec.Contains("pgs", StringComparison.OrdinalIgnoreCase)) + { + return ".sup"; + } + else + { + return ".srt"; + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index f4ebac377..a674aa928 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,7 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; -using MediaBrowser.Common; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -16,12 +16,15 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to move trickplay files to the new directory. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T23:00:00", nameof(MoveTrickplayFiles), RunMigrationOnSetup = true)] public class MoveTrickplayFiles : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; private readonly ILibraryManager _libraryManager; - private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IStartupLogger _logger; /// <summary> /// Initializes a new instance of the <see cref="MoveTrickplayFiles"/> class. @@ -30,7 +33,11 @@ public class MoveTrickplayFiles : IMigrationRoutine /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> /// <param name="logger">The logger.</param> - public MoveTrickplayFiles(ITrickplayManager trickplayManager, IFileSystem fileSystem, ILibraryManager libraryManager, ILogger<MoveTrickplayFiles> logger) + public MoveTrickplayFiles( + ITrickplayManager trickplayManager, + IFileSystem fileSystem, + ILibraryManager libraryManager, + IStartupLogger logger) { _trickplayManager = trickplayManager; _fileSystem = fileSystem; @@ -39,18 +46,9 @@ public class MoveTrickplayFiles : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("9540D44A-D8DC-11EF-9CBB-B77274F77C52"); - - /// <inheritdoc /> - public string Name => "MoveTrickplayFiles"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { - const int Limit = 100; + const int Limit = 5000; int itemCount = 0, offset = 0, previousCount; var sw = Stopwatch.StartNew(); @@ -65,9 +63,6 @@ public class MoveTrickplayFiles : IMigrationRoutine do { var trickplayInfos = _trickplayManager.GetTrickplayItemsAsync(Limit, offset).GetAwaiter().GetResult(); - previousCount = trickplayInfos.Count; - offset += Limit; - trickplayQuery.ItemIds = trickplayInfos.Select(i => i.ItemId).Distinct().ToArray(); var items = _libraryManager.GetItemList(trickplayQuery); foreach (var trickplayInfo in trickplayInfos) @@ -78,24 +73,32 @@ public class MoveTrickplayFiles : IMigrationRoutine continue; } - if (++itemCount % 1_000 == 0) - { - _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); - } - + var moved = false; var oldPath = GetOldTrickplayDirectory(item, trickplayInfo.Width); var newPath = _trickplayManager.GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); if (_fileSystem.DirectoryExists(oldPath)) { _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; } oldPath = GetNewOldTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); if (_fileSystem.DirectoryExists(oldPath)) { _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; + } + + if (moved) + { + itemCount++; } } + + offset += Limit; + previousCount = trickplayInfos.Count; + + _logger.LogInformation("Checked: {Checked} - Moved: {Count} - Time: {Time}", offset, itemCount, sw.Elapsed); } while (previousCount == Limit); _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); diff --git a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs index 9cfaec46f..ebf4a2780 100644 --- a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs @@ -2,48 +2,41 @@ using System; using MediaBrowser.Controller.Configuration; using MediaBrowser.Model.Updates; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to initialize system configuration with the default plugin repository. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T11:00:00", nameof(ReaddDefaultPluginRepository), "5F86E7F6-D966-4C77-849D-7A7B40B68C4E", RunMigrationOnSetup = true)] +public class ReaddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private readonly IServerConfigurationManager _serverConfigurationManager; + + private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo + { + Name = "Jellyfin Stable", + Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" + }; + /// <summary> - /// Migration to initialize system configuration with the default plugin repository. + /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. /// </summary> - public class ReaddDefaultPluginRepository : IMigrationRoutine + /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) { - private readonly IServerConfigurationManager _serverConfigurationManager; - - private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo - { - Name = "Jellyfin Stable", - Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" - }; - - /// <summary> - /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. - /// </summary> - /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> - public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) - { - _serverConfigurationManager = serverConfigurationManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("5F86E7F6-D966-4C77-849D-7A7B40B68C4E"); - - /// <inheritdoc/> - public string Name => "ReaddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; + _serverConfigurationManager = serverConfigurationManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + // Only add if repository list is empty + if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) { - // Only add if repository list is empty - if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) - { - _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; - _serverConfigurationManager.SaveConfiguration(); - } + _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; + _serverConfigurationManager.SaveConfiguration(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs new file mode 100644 index 000000000..b23a7dbc4 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to re-read creation dates for library items with internal metadata paths. +/// </summary> +[JellyfinMigration("2025-04-20T23:00:00", nameof(RefreshInternalDateModified))] +public class RefreshInternalDateModified : IDatabaseMigrationRoutine +{ + private readonly ILogger<RefreshInternalDateModified> _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IFileSystem _fileSystem; + private readonly IServerApplicationHost _applicationHost; + private readonly bool _useFileCreationTimeForDateAdded; + + private IReadOnlyList<string> _internalTypes = [ + typeof(Genre).FullName!, + typeof(MusicGenre).FullName!, + typeof(MusicArtist).FullName!, + typeof(People).FullName!, + typeof(Studio).FullName! + ]; + + private IReadOnlyList<string> _internalPaths; + + /// <summary> + /// Initializes a new instance of the <see cref="RefreshInternalDateModified"/> class. + /// </summary> + /// <param name="applicationHost">Instance of the <see cref="IServerApplicationHost"/> interface.</param> + /// <param name="applicationPaths">Instance of the <see cref="IServerApplicationPaths"/> interface.</param> + /// <param name="configurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + public RefreshInternalDateModified( + IServerApplicationHost applicationHost, + IServerApplicationPaths applicationPaths, + IServerConfigurationManager configurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + ILogger<RefreshInternalDateModified> logger, + IFileSystem fileSystem) + { + _dbProvider = dbProvider; + _logger = logger; + _fileSystem = fileSystem; + _applicationHost = applicationHost; + _internalPaths = [ + applicationPaths.ArtistsPath, + applicationPaths.GenrePath, + applicationPaths.MusicGenrePath, + applicationPaths.StudioPath, + applicationPaths.PeoplePath + ]; + _useFileCreationTimeForDateAdded = configurationManager.GetMetadataConfiguration().UseFileCreationTimeForDateAdded; + } + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => _internalTypes.Contains(b.Type)); + _logger.LogInformation("Checking if {Count} potentially internal items require refreshed DateModified", records); + + do + { + var results = context.BaseItems + .Where(b => _internalTypes.Contains(b.Type)) + .OrderBy(e => e.Id) + .Skip(offset) + .Take(Limit) + .ToList(); + + foreach (var item in results) + { + var itemPath = item.Path; + if (itemPath is not null) + { + var realPath = _applicationHost.ExpandVirtualPath(item.Path); + if (_internalPaths.Any(path => realPath.StartsWith(path, StringComparison.Ordinal))) + { + var writeTime = _fileSystem.GetLastWriteTimeUtc(realPath); + var itemModificationTime = item.DateModified; + if (writeTime != itemModificationTime) + { + _logger.LogDebug("Reset file modification date: Old: {Old} - New: {New} - Path: {Path}", itemModificationTime, writeTime, realPath); + item.DateModified = writeTime; + if (_useFileCreationTimeForDateAdded) + { + item.DateCreated = _fileSystem.GetCreationTimeUtc(realPath); + } + + itemCount++; + } + } + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Refreshed: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + + _logger.LogInformation("Refreshed DateModified for {Count} items in {Time}", itemCount, sw.Elapsed); + } +} diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs index 52fb93d59..b626c473e 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs @@ -3,50 +3,43 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Removes the old 'RemoveDownloadImagesInAdvance' from library options. - /// </summary> - internal class RemoveDownloadImagesInAdvance : IMigrationRoutine - { - private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; - private readonly ILibraryManager _libraryManager; - - public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) - { - _logger = logger; - _libraryManager = libraryManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{A81F75E0-8F43-416F-A5E8-516CCAB4D8CC}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "RemoveDownloadImagesInAdvance"; +/// <summary> +/// Removes the old 'RemoveDownloadImagesInAdvance' from library options. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T13:00:00", nameof(RemoveDownloadImagesInAdvance), "A81F75E0-8F43-416F-A5E8-516CCAB4D8CC")] +internal class RemoveDownloadImagesInAdvance : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete +{ + private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; + private readonly ILibraryManager _libraryManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) + { + _logger = logger; + _libraryManager = libraryManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + var virtualFolders = _libraryManager.GetVirtualFolders(false); + _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); + foreach (var virtualFolder in virtualFolders) { - var virtualFolders = _libraryManager.GetVirtualFolders(false); - _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); - foreach (var virtualFolder in virtualFolders) + // Some virtual folders don't have a proper item id. + if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) { - // Some virtual folders don't have a proper item id. - if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) - { - continue; - } - - var libraryOptions = virtualFolder.LibraryOptions; - var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); - // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. - collectionFolder.UpdateLibraryOptions(libraryOptions); - _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); + continue; } + + var libraryOptions = virtualFolder.LibraryOptions; + var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); + // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. + collectionFolder.UpdateLibraryOptions(libraryOptions); + _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs index 7b0d9456d..c9e66d0cf 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs @@ -7,77 +7,70 @@ using MediaBrowser.Controller; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T08:00:00", nameof(RemoveDuplicateExtras), "ACBE17B7-8435-4A83-8B64-6FCF162CB9BD")] +internal class RemoveDuplicateExtras : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - /// <summary> - /// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. - /// </summary> - internal class RemoveDuplicateExtras : IMigrationRoutine + private const string DbFilename = "library.db"; + private readonly ILogger<RemoveDuplicateExtras> _logger; + private readonly IServerApplicationPaths _paths; + + public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) { - private const string DbFilename = "library.db"; - private readonly ILogger<RemoveDuplicateExtras> _logger; - private readonly IServerApplicationPaths _paths; + _logger = logger; + _paths = paths; + } - public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + var dbPath = Path.Combine(dataPath, DbFilename); + using var connection = new SqliteConnection($"Filename={dbPath}"); + connection.Open(); + using (var transaction = connection.BeginTransaction()) { - _logger = logger; - _paths = paths; - } + // Query the database for the ids of duplicate extras + var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); + var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - /// <inheritdoc/> - public Guid Id => Guid.Parse("{ACBE17B7-8435-4A83-8B64-6FCF162CB9BD}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicateExtras"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - var dbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + // Do nothing if no duplicate extras were detected + if (bads.Length == 0) { - // Query the database for the ids of duplicate extras - var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); - var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - - // Do nothing if no duplicate extras were detected - if (bads.Length == 0) - { - _logger.LogInformation("No duplicate extras detected, skipping migration."); - return; - } + _logger.LogInformation("No duplicate extras detected, skipping migration."); + return; + } - // Back up the database before deleting any entries - for (int i = 1; ; i++) + // Back up the database before deleting any entries + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); + if (!File.Exists(bakPath)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + try { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } + File.Copy(dbPath, bakPath); + _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; } } - - // Delete all duplicate extras - _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); - connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); - transaction.Commit(); } + + // Delete all duplicate extras + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); + connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); + transaction.Commit(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs index f84bccc25..23f212424 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs @@ -1,44 +1,33 @@ using System; using System.Linq; using System.Threading; - using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Playlists; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Remove duplicate playlist entries. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T19:00:00", nameof(RemoveDuplicatePlaylistChildren), "96C156A2-7A13-4B3B-A8B8-FB80C94D20C0")] internal class RemoveDuplicatePlaylistChildren : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private readonly ILogger<RemoveDuplicatePlaylistChildren> _logger; private readonly ILibraryManager _libraryManager; private readonly IPlaylistManager _playlistManager; public RemoveDuplicatePlaylistChildren( - ILogger<RemoveDuplicatePlaylistChildren> logger, ILibraryManager libraryManager, IPlaylistManager playlistManager) { - _logger = logger; _libraryManager = libraryManager; _playlistManager = playlistManager; } /// <inheritdoc/> - public Guid Id => Guid.Parse("{96C156A2-7A13-4B3B-A8B8-FB80C94D20C0}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicatePlaylistChildren"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs index 7e8c8ac87..f58cf2741 100644 --- a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs @@ -6,7 +6,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to update the default Jellyfin plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T17:00:00", nameof(UpdateDefaultPluginRepository), "852816E0-2712-49A9-9240-C6FC5FCAD1A8", RunMigrationOnSetup = true)] public class UpdateDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string NewRepositoryUrl = "https://repo.jellyfin.org/files/plugin/manifest.json"; private const string OldRepositoryUrl = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json"; @@ -23,15 +26,6 @@ public class UpdateDefaultPluginRepository : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("852816E0-2712-49A9-9240-C6FC5FCAD1A8"); - - /// <inheritdoc /> - public string Name => "UpdateDefaultPluginRepository10.9"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { var updated = false; diff --git a/Jellyfin.Server/Migrations/Stages/CodeMigration.cs b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs new file mode 100644 index 000000000..47ed26965 --- /dev/null +++ b/Jellyfin.Server/Migrations/Stages/CodeMigration.cs @@ -0,0 +1,81 @@ +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace Jellyfin.Server.Migrations.Stages; + +internal class CodeMigration(Type migrationType, JellyfinMigrationAttribute metadata, JellyfinMigrationBackupAttribute? migrationBackupAttribute) +{ + public Type MigrationType { get; } = migrationType; + + public JellyfinMigrationAttribute Metadata { get; } = metadata; + + public JellyfinMigrationBackupAttribute? BackupRequirements { get; set; } = migrationBackupAttribute; + + public string BuildCodeMigrationId() + { + return Metadata.Order.ToString("yyyyMMddHHmmsss", CultureInfo.InvariantCulture) + "_" + Metadata.Name!; + } + + private ServiceCollection MigrationServices(IServiceProvider serviceProvider, IStartupLogger logger) + { + var childServiceCollection = new ServiceCollection(); + childServiceCollection.AddSingleton(serviceProvider); + childServiceCollection.AddSingleton(logger); + + foreach (ServiceDescriptor service in serviceProvider.GetRequiredService<IServiceCollection>()) + { + if (service.Lifetime == ServiceLifetime.Singleton && !service.ServiceType.IsGenericTypeDefinition) + { + object? serviceInstance = serviceProvider.GetService(service.ServiceType); + if (serviceInstance != null) + { + childServiceCollection.AddSingleton(service.ServiceType, serviceInstance); + continue; + } + } + + childServiceCollection.Add(service); + } + + return childServiceCollection; + } + + public async Task Perform(IServiceProvider? serviceProvider, IStartupLogger logger, CancellationToken cancellationToken) + { +#pragma warning disable CS0618 // Type or member is obsolete + if (typeof(IMigrationRoutine).IsAssignableFrom(MigrationType)) + { + if (serviceProvider is null) + { + ((IMigrationRoutine)Activator.CreateInstance(MigrationType)!).Perform(); + } + else + { + using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); + ((IMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).Perform(); +#pragma warning restore CS0618 // Type or member is obsolete + } + } + else if (typeof(IAsyncMigrationRoutine).IsAssignableFrom(MigrationType)) + { + if (serviceProvider is null) + { + await ((IAsyncMigrationRoutine)Activator.CreateInstance(MigrationType)!).PerformAsync(cancellationToken).ConfigureAwait(false); + } + else + { + using var migrationServices = MigrationServices(serviceProvider, logger).BuildServiceProvider(); + await ((IAsyncMigrationRoutine)ActivatorUtilities.CreateInstance(migrationServices, MigrationType)).PerformAsync(cancellationToken).ConfigureAwait(false); + } + } + else + { + throw new InvalidOperationException($"The type {MigrationType} does not implement either IMigrationRoutine or IAsyncMigrationRoutine and is not a valid migration type"); + } + } +} diff --git a/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs new file mode 100644 index 000000000..3d5ec233b --- /dev/null +++ b/Jellyfin.Server/Migrations/Stages/JellyfinMigrationStageTypes.cs @@ -0,0 +1,26 @@ +namespace Jellyfin.Server.Migrations.Stages; + +/// <summary> +/// Defines the stages the <see cref="JellyfinMigrationService"/> supports. +/// </summary> +#pragma warning disable CA1008 // Enums should have zero value +public enum JellyfinMigrationStageTypes +#pragma warning restore CA1008 // Enums should have zero value +{ + /// <summary> + /// Runs before services are initialised. + /// Reserved for migrations that are modifying the application server itself. Should be avoided if possible. + /// </summary> + PreInitialisation = 1, + + /// <summary> + /// Runs after the host has been configured and includes the database migrations. + /// Allows the mix order of migrations that contain application code and database changes. + /// </summary> + CoreInitialisation = 2, + + /// <summary> + /// Runs after services has been registered and initialised. Last step before running the server. + /// </summary> + AppInitialisation = 3 +} diff --git a/Jellyfin.Server/Migrations/Stages/MigrationStage.cs b/Jellyfin.Server/Migrations/Stages/MigrationStage.cs new file mode 100644 index 000000000..efcadbf00 --- /dev/null +++ b/Jellyfin.Server/Migrations/Stages/MigrationStage.cs @@ -0,0 +1,16 @@ +using System.Collections.ObjectModel; + +namespace Jellyfin.Server.Migrations.Stages; + +/// <summary> +/// Defines a Stage that can be Invoked and Handled at different times from the code. +/// </summary> +internal class MigrationStage : Collection<CodeMigration> +{ + public MigrationStage(JellyfinMigrationStageTypes stage) + { + Stage = stage; + } + + public JellyfinMigrationStageTypes Stage { get; } +} diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index e661d0d4a..0b77d63ac 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -8,15 +9,22 @@ using System.Threading; using System.Threading.Tasks; using CommandLine; using Emby.Server.Implementations; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; using Jellyfin.Database.Implementations; using Jellyfin.Server.Extensions; using Jellyfin.Server.Helpers; +using Jellyfin.Server.Implementations.DatabaseConfiguration; +using Jellyfin.Server.Implementations.Extensions; +using Jellyfin.Server.Implementations.StorageHelpers; +using Jellyfin.Server.Implementations.SystemBackupService; +using Jellyfin.Server.Migrations; +using Jellyfin.Server.Migrations.Stages; using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller; using Microsoft.AspNetCore.Hosting; -using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; @@ -46,12 +54,14 @@ namespace Jellyfin.Server public const string LoggingConfigFileSystem = "logging.json"; private static readonly SerilogLoggerFactory _loggerFactory = new SerilogLoggerFactory(); - private static SetupServer _setupServer = new(); + private static SetupServer? _setupServer; private static CoreAppHost? _appHost; private static IHost? _jellyfinHost = null; private static long _startTimestamp; private static ILogger _logger = NullLogger.Instance; private static bool _restartOnShutdown; + private static IStartupLogger? _migrationLogger; + private static string? _restoreFromBackup; /// <summary> /// The entry point of the application. @@ -73,9 +83,10 @@ namespace Jellyfin.Server private static async Task StartApp(StartupOptions options) { + _restoreFromBackup = options.RestoreArchive; _startTimestamp = Stopwatch.GetTimestamp(); ServerApplicationPaths appPaths = StartupHelpers.CreateApplicationPaths(options); - await _setupServer.RunAsync(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost).ConfigureAwait(false); + appPaths.MakeSanityCheckOrThrow(); // $JELLYFIN_LOG_DIR needs to be set for the logger configuration manager Environment.SetEnvironmentVariable("JELLYFIN_LOG_DIR", appPaths.LogDirectoryPath); @@ -88,8 +99,9 @@ namespace Jellyfin.Server // Create an instance of the application configuration to use for application startup IConfiguration startupConfig = CreateAppConfiguration(options, appPaths); - StartupHelpers.InitializeLoggingFramework(startupConfig, appPaths); + _setupServer = new SetupServer(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost, _loggerFactory, startupConfig); + await _setupServer.RunAsync().ConfigureAwait(false); _logger = _loggerFactory.CreateLogger("Main"); // Use the logging framework for uncaught exceptions instead of std error @@ -120,8 +132,11 @@ namespace Jellyfin.Server } } + StorageHelper.TestCommonPathsForStorageCapacity(appPaths, StartupLogger.Logger.With(_loggerFactory.CreateLogger<Startup>()).BeginGroup($"Storage Check")); + StartupHelpers.PerformStaticInitialization(); - await Migrations.MigrationRunner.RunPreStartup(appPaths, _loggerFactory).ConfigureAwait(false); + + await ApplyStartupMigrationAsync(appPaths, startupConfig).ConfigureAwait(false); do { @@ -130,10 +145,12 @@ namespace Jellyfin.Server if (_restartOnShutdown) { _startTimestamp = Stopwatch.GetTimestamp(); - _setupServer = new SetupServer(); - await _setupServer.RunAsync(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost).ConfigureAwait(false); + await _setupServer.StopAsync().ConfigureAwait(false); + await _setupServer.RunAsync().ConfigureAwait(false); } } while (_restartOnShutdown); + + _setupServer.Dispose(); } private static async Task StartServer(IServerApplicationPaths appPaths, StartupOptions options, IConfiguration startupConfig) @@ -144,6 +161,7 @@ namespace Jellyfin.Server options, startupConfig); _appHost = appHost; + var configurationCompleted = false; try { _jellyfinHost = Host.CreateDefaultBuilder() @@ -160,19 +178,33 @@ namespace Jellyfin.Server }) .ConfigureAppConfiguration(config => config.ConfigureAppConfiguration(options, appPaths, startupConfig)) .UseSerilog() + .ConfigureServices(e => e.AddTransient<IStartupLogger, StartupLogger>().AddSingleton<IServiceCollection>(e)) .Build(); // Re-use the host service provider in the app host since ASP.NET doesn't allow a custom service collection. appHost.ServiceProvider = _jellyfinHost.Services; + PrepareDatabaseProvider(appHost.ServiceProvider); + + if (!string.IsNullOrWhiteSpace(_restoreFromBackup)) + { + await appHost.ServiceProvider.GetService<IBackupService>()!.RestoreBackupAsync(_restoreFromBackup).ConfigureAwait(false); + _restoreFromBackup = null; + _restartOnShutdown = true; + return; + } + + var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(appHost.ServiceProvider); + await jellyfinMigrationService.PrepareSystemForMigration(_logger).ConfigureAwait(false); + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.CoreInitialisation, appHost.ServiceProvider).ConfigureAwait(false); await appHost.InitializeServices(startupConfig).ConfigureAwait(false); - await Migrations.MigrationRunner.Run(appHost, _loggerFactory).ConfigureAwait(false); + await jellyfinMigrationService.MigrateStepAsync(JellyfinMigrationStageTypes.AppInitialisation, appHost.ServiceProvider).ConfigureAwait(false); + await jellyfinMigrationService.CleanupSystemAfterMigration(_logger).ConfigureAwait(false); try { - await _setupServer.StopAsync().ConfigureAwait(false); - _setupServer.Dispose(); - _setupServer = null!; + configurationCompleted = true; + await _setupServer!.StopAsync().ConfigureAwait(false); await _jellyfinHost.StartAsync().ConfigureAwait(false); if (!OperatingSystem.IsWindows() && startupConfig.UseUnixSocket()) @@ -194,11 +226,18 @@ namespace Jellyfin.Server await _jellyfinHost.WaitForShutdownAsync().ConfigureAwait(false); _restartOnShutdown = appHost.ShouldRestart; + _restoreFromBackup = appHost.RestoreBackupPath; } catch (Exception ex) { _restartOnShutdown = false; _logger.LogCritical(ex, "Error while starting server"); + if (_setupServer!.IsAlive && !configurationCompleted) + { + _setupServer!.SoftStop(); + await Task.Delay(TimeSpan.FromMinutes(10)).ConfigureAwait(false); + await _setupServer!.StopAsync().ConfigureAwait(false); + } } finally { @@ -219,6 +258,52 @@ namespace Jellyfin.Server } /// <summary> + /// [Internal]Runs the startup Migrations. + /// </summary> + /// <remarks> + /// Not intended to be used other then by jellyfin and its tests. + /// </remarks> + /// <param name="appPaths">Application Paths.</param> + /// <param name="startupConfig">Startup Config.</param> + /// <returns>A task.</returns> + public static async Task ApplyStartupMigrationAsync(ServerApplicationPaths appPaths, IConfiguration startupConfig) + { + _migrationLogger = StartupLogger.Logger.BeginGroup($"Migration Service"); + var startupConfigurationManager = new ServerConfigurationManager(appPaths, _loggerFactory, new MyXmlSerializer()); + startupConfigurationManager.AddParts([new DatabaseConfigurationFactory()]); + var migrationStartupServiceProvider = new ServiceCollection() + .AddLogging(d => d.AddSerilog()) + .AddJellyfinDbContext(startupConfigurationManager, startupConfig) + .AddSingleton<IApplicationPaths>(appPaths) + .AddSingleton<ServerApplicationPaths>(appPaths) + .AddSingleton<IStartupLogger>(_migrationLogger); + + migrationStartupServiceProvider.AddSingleton(migrationStartupServiceProvider); + var startupService = migrationStartupServiceProvider.BuildServiceProvider(); + + PrepareDatabaseProvider(startupService); + + var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(startupService); + await jellyfinMigrationService.CheckFirstTimeRunOrMigration(appPaths).ConfigureAwait(false); + await jellyfinMigrationService.MigrateStepAsync(Migrations.Stages.JellyfinMigrationStageTypes.PreInitialisation, startupService).ConfigureAwait(false); + } + + /// <summary> + /// [Internal]Runs the Jellyfin migrator service with the Core stage. + /// </summary> + /// <remarks> + /// Not intended to be used other then by jellyfin and its tests. + /// </remarks> + /// <param name="serviceProvider">The service provider.</param> + /// <param name="jellyfinMigrationStage">The stage to run.</param> + /// <returns>A task.</returns> + public static async Task ApplyCoreMigrationsAsync(IServiceProvider serviceProvider, Migrations.Stages.JellyfinMigrationStageTypes jellyfinMigrationStage) + { + var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(serviceProvider, _migrationLogger!); + await jellyfinMigrationService.MigrateStepAsync(jellyfinMigrationStage, serviceProvider).ConfigureAwait(false); + } + + /// <summary> /// Create the application configuration. /// </summary> /// <param name="commandLineOpts">The command line options passed to the program.</param> @@ -252,5 +337,12 @@ namespace Jellyfin.Server .AddEnvironmentVariables("JELLYFIN_") .AddInMemoryCollection(commandLineOpts.ConvertToConfig()); } + + private static void PrepareDatabaseProvider(IServiceProvider services) + { + var factory = services.GetRequiredService<IDbContextFactory<JellyfinDbContext>>(); + var provider = services.GetRequiredService<IJellyfinDatabaseProvider>(); + provider.DbContextFactory = factory; + } } } diff --git a/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs b/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs new file mode 100644 index 000000000..2c2ef05f8 --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/IStartupLogger.cs @@ -0,0 +1,25 @@ +using System; +using Morestachio.Helper.Logging; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Jellyfin.Server.ServerSetupApp; + +/// <summary> +/// Defines the Startup Logger. This logger acts an an aggregate logger that will push though all log messages to both the attached logger as well as the startup UI. +/// </summary> +public interface IStartupLogger : ILogger +{ + /// <summary> + /// Adds another logger instance to this logger for combined logging. + /// </summary> + /// <param name="logger">Other logger to rely messages to.</param> + /// <returns>A combined logger.</returns> + IStartupLogger With(ILogger logger); + + /// <summary> + /// Opens a new Group logger within the parent logger. + /// </summary> + /// <param name="logEntry">Defines the log message that introduces the new group.</param> + /// <returns>A new logger that can write to the group.</returns> + IStartupLogger BeginGroup(FormattableString logEntry); +} diff --git a/Jellyfin.Server/ServerSetupApp/SetupServer.cs b/Jellyfin.Server/ServerSetupApp/SetupServer.cs index 9e2cf5bc8..d88dbee57 100644 --- a/Jellyfin.Server/ServerSetupApp/SetupServer.cs +++ b/Jellyfin.Server/ServerSetupApp/SetupServer.cs @@ -1,19 +1,32 @@ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; +using Jellyfin.Networking.Manager; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller; +using MediaBrowser.Model.IO; using MediaBrowser.Model.System; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using Morestachio; +using Morestachio.Framework.IO.SingleStream; +using Morestachio.Rendering; namespace Jellyfin.Server.ServerSetupApp; @@ -22,22 +35,109 @@ namespace Jellyfin.Server.ServerSetupApp; /// </summary> public sealed class SetupServer : IDisposable { + private readonly Func<INetworkManager?> _networkManagerFactory; + private readonly IApplicationPaths _applicationPaths; + private readonly Func<IServerApplicationHost?> _serverFactory; + private readonly ILoggerFactory _loggerFactory; + private readonly IConfiguration _startupConfiguration; + private readonly ServerConfigurationManager _configurationManager; + private IRenderer? _startupUiRenderer; private IHost? _startupServer; private bool _disposed; + private bool _isUnhealthy; /// <summary> - /// Starts the Bind-All Setup aspcore server to provide a reflection on the current core setup. + /// Initializes a new instance of the <see cref="SetupServer"/> class. /// </summary> /// <param name="networkManagerFactory">The networkmanager.</param> /// <param name="applicationPaths">The application paths.</param> - /// <param name="serverApplicationHost">The servers application host.</param> - /// <returns>A Task.</returns> - public async Task RunAsync( + /// <param name="serverApplicationHostFactory">The servers application host.</param> + /// <param name="loggerFactory">The logger factory.</param> + /// <param name="startupConfiguration">The startup configuration.</param> + public SetupServer( Func<INetworkManager?> networkManagerFactory, IApplicationPaths applicationPaths, - Func<IServerApplicationHost?> serverApplicationHost) + Func<IServerApplicationHost?> serverApplicationHostFactory, + ILoggerFactory loggerFactory, + IConfiguration startupConfiguration) + { + _networkManagerFactory = networkManagerFactory; + _applicationPaths = applicationPaths; + _serverFactory = serverApplicationHostFactory; + _loggerFactory = loggerFactory; + _startupConfiguration = startupConfiguration; + var xmlSerializer = new MyXmlSerializer(); + _configurationManager = new ServerConfigurationManager(_applicationPaths, loggerFactory, xmlSerializer); + _configurationManager.RegisterConfiguration<NetworkConfigurationFactory>(); + } + + internal static ConcurrentQueue<StartupLogEntry>? LogQueue { get; set; } = new(); + + /// <summary> + /// Gets a value indicating whether Startup server is currently running. + /// </summary> + public bool IsAlive { get; internal set; } + + /// <summary> + /// Starts the Bind-All Setup aspcore server to provide a reflection on the current core setup. + /// </summary> + /// <returns>A Task.</returns> + public async Task RunAsync() { + var fileTemplate = await File.ReadAllTextAsync(Path.Combine(AppContext.BaseDirectory, "ServerSetupApp", "index.mstemplate.html")).ConfigureAwait(false); + _startupUiRenderer = (await ParserOptionsBuilder.New() + .WithTemplate(fileTemplate) + .WithFormatter( + (StartupLogEntry logEntry, IEnumerable<StartupLogEntry> children) => + { + if (children.Any()) + { + var maxLevel = logEntry.LogLevel; + var stack = new Stack<StartupLogEntry>(children); + + while (maxLevel != LogLevel.Error && stack.Count > 0 && (logEntry = stack.Pop()) != null) // error is the highest inherted error level. + { + maxLevel = maxLevel < logEntry.LogLevel ? logEntry.LogLevel : maxLevel; + foreach (var child in logEntry.Children) + { + stack.Push(child); + } + } + + return maxLevel; + } + + return logEntry.LogLevel; + }, + "FormatLogLevel") + .WithFormatter( + (LogLevel logLevel) => + { + switch (logLevel) + { + case LogLevel.Trace: + case LogLevel.Debug: + case LogLevel.None: + return "success"; + case LogLevel.Information: + return "info"; + case LogLevel.Warning: + return "warn"; + case LogLevel.Error: + return "danger"; + case LogLevel.Critical: + return "danger-strong"; + } + + return string.Empty; + }, + "ToString") + .BuildAndParseAsync() + .ConfigureAwait(false)) + .CreateCompiledRenderer(); + ThrowIfDisposed(); + var retryAfterValue = TimeSpan.FromSeconds(5); _startupServer = Host.CreateDefaultBuilder() .UseConsoleLifetime() .ConfigureServices(serv => @@ -48,7 +148,23 @@ public sealed class SetupServer : IDisposable .ConfigureWebHostDefaults(webHostBuilder => { webHostBuilder - .UseKestrel() + .UseKestrel((builderContext, options) => + { + var config = _configurationManager.GetNetworkConfiguration()!; + var knownBindInterfaces = NetworkManager.GetInterfacesCore(_loggerFactory.CreateLogger<SetupServer>(), config.EnableIPv4, config.EnableIPv6); + knownBindInterfaces = NetworkManager.FilterBindSettings(config, knownBindInterfaces.ToList(), config.EnableIPv4, config.EnableIPv6); + var bindInterfaces = NetworkManager.GetAllBindInterfaces(false, _configurationManager, knownBindInterfaces, config.EnableIPv4, config.EnableIPv6); + Extensions.WebHostBuilderExtensions.SetupJellyfinWebServer( + bindInterfaces, + config.InternalHttpPort, + null, + null, + _startupConfiguration, + _applicationPaths, + _loggerFactory.CreateLogger<SetupServer>(), + builderContext, + options); + }) .Configure(app => { app.UseHealthChecks("/health"); @@ -57,16 +173,16 @@ public sealed class SetupServer : IDisposable { loggerRoute.Run(async context => { - var networkManager = networkManagerFactory(); + var networkManager = _networkManagerFactory(); if (context.Connection.RemoteIpAddress is null || networkManager is null || !networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress)) { context.Response.StatusCode = (int)HttpStatusCode.Unauthorized; return; } - var logFilePath = new DirectoryInfo(applicationPaths.LogDirectoryPath) + var logFilePath = new DirectoryInfo(_applicationPaths.LogDirectoryPath) .EnumerateFiles() - .OrderBy(f => f.CreationTimeUtc) + .OrderByDescending(f => f.CreationTimeUtc) .FirstOrDefault() ?.FullName; if (logFilePath is not null) @@ -80,20 +196,20 @@ public sealed class SetupServer : IDisposable { systemRoute.Run(async context => { - var jfApplicationHost = serverApplicationHost(); + var jfApplicationHost = _serverFactory(); var retryCounter = 0; while (jfApplicationHost is null && retryCounter < 5) { await Task.Delay(500).ConfigureAwait(false); - jfApplicationHost = serverApplicationHost(); + jfApplicationHost = _serverFactory(); retryCounter++; } if (jfApplicationHost is null) { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new Microsoft.Extensions.Primitives.StringValues("60"); + context.Response.Headers.RetryAfter = new StringValues(retryAfterValue.TotalSeconds.ToString("000", CultureInfo.InvariantCulture)); return; } @@ -111,23 +227,30 @@ public sealed class SetupServer : IDisposable }); }); - app.Run((context) => + app.Run(async (context) => { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new Microsoft.Extensions.Primitives.StringValues("60"); - context.Response.WriteAsync("<p>Jellyfin Server still starting. Please wait.</p>"); - var networkManager = networkManagerFactory(); - if (networkManager is not null && context.Connection.RemoteIpAddress is not null && networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress)) - { - context.Response.WriteAsync("<p>You can download the current logfiles <a href='/startup/logger'>here</a>.</p>"); - } + context.Response.Headers.RetryAfter = new StringValues(retryAfterValue.TotalSeconds.ToString("000", CultureInfo.InvariantCulture)); + context.Response.Headers.ContentType = new StringValues("text/html"); + var networkManager = _networkManagerFactory(); - return Task.CompletedTask; + var startupLogEntries = LogQueue?.ToArray() ?? []; + await _startupUiRenderer.RenderAsync( + new Dictionary<string, object>() + { + { "isInReportingMode", _isUnhealthy }, + { "retryValue", retryAfterValue }, + { "logs", startupLogEntries }, + { "localNetworkRequest", networkManager is not null && context.Connection.RemoteIpAddress is not null && networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress) } + }, + new ByteCounterStream(context.Response.BodyWriter.AsStream(), IODefaults.FileStreamBufferSize, true, _startupUiRenderer.ParserOptions)) + .ConfigureAwait(false); }); }); }) .Build(); await _startupServer.StartAsync().ConfigureAwait(false); + IsAlive = true; } /// <summary> @@ -143,6 +266,7 @@ public sealed class SetupServer : IDisposable } await _startupServer.StopAsync().ConfigureAwait(false); + IsAlive = false; } /// <inheritdoc/> @@ -155,6 +279,9 @@ public sealed class SetupServer : IDisposable _disposed = true; _startupServer?.Dispose(); + IsAlive = false; + LogQueue?.Clear(); + LogQueue = null; } private void ThrowIfDisposed() @@ -162,11 +289,88 @@ public sealed class SetupServer : IDisposable ObjectDisposedException.ThrowIf(_disposed, this); } + internal void SoftStop() + { + _isUnhealthy = true; + } + private class SetupHealthcheck : IHealthCheck { + private readonly SetupServer _startupServer; + + public SetupHealthcheck(SetupServer startupServer) + { + _startupServer = startupServer; + } + public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) { + if (_startupServer._isUnhealthy) + { + return Task.FromResult(HealthCheckResult.Unhealthy("Server is could not complete startup. Check logs.")); + } + return Task.FromResult(HealthCheckResult.Degraded("Server is still starting up.")); } } + + internal sealed class SetupLoggerFactory : ILoggerProvider, IDisposable + { + private bool _disposed; + + public ILogger CreateLogger(string categoryName) + { + return new CatchingSetupServerLogger(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + } + } + + internal sealed class CatchingSetupServerLogger : ILogger + { + public IDisposable? BeginScope<TState>(TState state) + where TState : notnull + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return logLevel is LogLevel.Error or LogLevel.Critical; + } + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + LogQueue?.Enqueue(new() + { + LogLevel = logLevel, + Content = formatter(state, exception), + DateOfCreation = DateTimeOffset.Now + }); + } + } + + internal class StartupLogEntry + { + public LogLevel LogLevel { get; set; } + + public string? Content { get; set; } + + public DateTimeOffset DateOfCreation { get; set; } + + public List<StartupLogEntry> Children { get; set; } = []; + } } diff --git a/Jellyfin.Server/ServerSetupApp/StartupLogger.cs b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs new file mode 100644 index 000000000..2b86dc0c1 --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/StartupLogger.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Jellyfin.Server.Migrations.Routines; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.ServerSetupApp; + +/// <inheritdoc/> +public class StartupLogger : IStartupLogger +{ + private readonly SetupServer.StartupLogEntry? _groupEntry; + + /// <summary> + /// Initializes a new instance of the <see cref="StartupLogger"/> class. + /// </summary> + public StartupLogger() + { + Loggers = []; + } + + /// <summary> + /// Initializes a new instance of the <see cref="StartupLogger"/> class. + /// </summary> + private StartupLogger(SetupServer.StartupLogEntry? groupEntry) : this() + { + _groupEntry = groupEntry; + } + + internal static IStartupLogger Logger { get; } = new StartupLogger(); + + private List<ILogger> Loggers { get; set; } + + /// <inheritdoc/> + public IStartupLogger BeginGroup(FormattableString logEntry) + { + var startupEntry = new SetupServer.StartupLogEntry() + { + Content = logEntry.ToString(CultureInfo.InvariantCulture), + DateOfCreation = DateTimeOffset.Now + }; + + if (_groupEntry is null) + { + SetupServer.LogQueue?.Enqueue(startupEntry); + } + else + { + _groupEntry.Children.Add(startupEntry); + } + + return new StartupLogger(startupEntry); + } + + /// <inheritdoc/> + public IDisposable? BeginScope<TState>(TState state) + where TState : notnull + { + return null; + } + + /// <inheritdoc/> + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + /// <inheritdoc/> + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + foreach (var item in Loggers.Where(e => e.IsEnabled(logLevel))) + { + item.Log(logLevel, eventId, state, exception, formatter); + } + + var startupEntry = new SetupServer.StartupLogEntry() + { + LogLevel = logLevel, + Content = formatter(state, exception), + DateOfCreation = DateTimeOffset.Now + }; + + if (_groupEntry is null) + { + SetupServer.LogQueue?.Enqueue(startupEntry); + } + else + { + _groupEntry.Children.Add(startupEntry); + } + } + + /// <inheritdoc/> + public IStartupLogger With(ILogger logger) + { + return new StartupLogger(_groupEntry) + { + Loggers = [.. Loggers, logger] + }; + } +} diff --git a/Jellyfin.Server/ServerSetupApp/index.mstemplate.html b/Jellyfin.Server/ServerSetupApp/index.mstemplate.html new file mode 100644 index 000000000..747835b2a --- /dev/null +++ b/Jellyfin.Server/ServerSetupApp/index.mstemplate.html @@ -0,0 +1,225 @@ +<!DOCTYPE html> +<html> + +<head> + <meta charset="UTF-8" /> + <title> + {{#IF isInReportingMode}} + ❌ + {{/IF}} + Jellyfin Startup + </title> + <style> + * { + font-family: sans-serif; + } + + .flex-row { + display: flex; + flex-direction: row; + flex-wrap: nowrap; + justify-content: center; + align-items: center; + align-content: normal; + } + + .flex-col { + display: flex; + flex-direction: column; + flex-wrap: nowrap; + justify-content: center; + align-items: center; + align-content: normal; + } + + header { + height: 5rem; + width: 100%; + } + + header svg { + height: 3rem; + width: 9rem; + margin-right: 1rem; + } + + /* ol.action-list { + list-style-type: none; + position: relative; + } */ + + ol.action-list * { + font-family: monospace; + font-weight: 300; + font-size: clamp(18px, 100vw / var(--width), 20px); + font-feature-settings: 'onum', 'pnum'; + line-height: 1.8; + -webkit-text-size-adjust: none; + } + + /* + ol.action-list li { + padding-top: .5rem; + } + + ol.action-list li::before { + position: absolute; + left: -0.8em; + font-size: 1.1em; + } */ + + /* Attribution as heavily inspired by: https://iamkate.com/code/tree-views/ */ + .action-list { + --spacing: 1.4rem; + --radius: 14px; + } + + .action-list li { + display: block; + position: relative; + padding-left: calc(2 * var(--spacing) - var(--radius) - 1px); + } + + .action-list ul { + margin-left: calc(var(--radius) - var(--spacing)); + padding-left: 0; + } + + .action-list ul li { + border-left: 2px solid #ddd; + } + + .action-list ul li:last-child { + border-color: transparent; + } + + .action-list ul li::before { + content: ''; + display: block; + position: absolute; + top: calc(var(--spacing) / -2); + left: -2px; + width: calc(var(--spacing) + 2px); + height: calc(var(--spacing) + 1px); + border: solid #ddd; + border-width: 0 0 2px 2px; + } + + .action-list summary { + display: block; + cursor: pointer; + } + + .action-list summary::marker, + .action-list summary::-webkit-details-marker { + display: none; + } + + .action-list summary:focus { + outline: none; + } + + .action-list summary:focus-visible { + outline: 1px dotted #000; + } + + .action-list li::after, + .action-list summary::before { + content: ''; + display: block; + position: absolute; + top: calc(var(--spacing) / 2 - var(--radius) + 4px); + left: calc(var(--spacing) - var(--radius) - -5px); + } + + .action-list summary::before { + z-index: 1; + /* background: #696 url('expand-collapse.svg') 0 0; */ + } + + .action-list details[open]>summary::before { + background-position: calc(-2 * var(--radius)) 0; + } + + .action-list li.danger-item::after, + .action-list li.danger-strong-item::after { + content: '❌'; + } + + ol.action-list li span.danger-strong-item { + text-decoration-style: solid; + text-decoration-color: red; + text-decoration-line: underline; + } + + ol.action-list li.warn-item::after { + content: '⚠️'; + } + + ol.action-list li.success-item::after { + content: '✅'; + } + + ol.action-list li.info-item::after { + content: '🔹'; + } + + /* End Attribution */ + </style> +</head> + +<body> + <div> + <header class="flex-row"> + + {{^IF isInReportingMode}} + <p>Jellyfin Server still starting. Please wait.</p> + {{#ELSE}} + <p>Jellyfin Server has encountered an error and was not able to start.</p> + {{/ELSE}} + {{/IF}} + + {{#IF localNetworkRequest}} + <p style="margin-left: 1rem;">You can download the current log file <a href='/startup/logger' + target="_blank">here</a>.</p> + {{/IF}} + </header> + + {{#DECLARE LogEntry |--}} + {{#LET children = Children}} + <li class="{{FormatLogLevel(children).ToString()}}-item"> + {{--| #IF children.Count > 0}} + <details open> + <summary>{{DateOfCreation}} - {{Content}}</summary> + <ul class="action-list"> + {{--| #EACH children.Reverse() |-}} + {{#IMPORT 'LogEntry'}} + {{--| /EACH |-}} + </ul> + </details> + {{--| #ELSE |-}} + <span class="{{FormatLogLevel(children).ToString()}}-item">{{DateOfCreation}} - {{Content}}</span> + {{--| /ELSE |--}} + {{--| /IF |-}} + </li> + {{--| /DECLARE}} + + <div class="flex-col"> + <ol class="action-list"> + {{#FOREACH log IN logs.Reverse()}} + {{#IMPORT 'LogEntry' #WITH log}} + {{/FOREACH}} + </ol> + </div> + </div> +</body> + +{{^IF isInReportingMode}} +<script> + setTimeout(() => { + window.location.reload(); + }, {{ retryValue.TotalMilliseconds }}); +</script> +{{/IF}} + +</html> diff --git a/Jellyfin.Server/Startup.cs b/Jellyfin.Server/Startup.cs index 688b16935..aa8f6dd1c 100644 --- a/Jellyfin.Server/Startup.cs +++ b/Jellyfin.Server/Startup.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -29,6 +30,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Primitives; using Prometheus; namespace Jellyfin.Server @@ -195,7 +197,14 @@ namespace Jellyfin.Server { FileProvider = new PhysicalFileProvider(_serverConfigurationManager.ApplicationPaths.WebPath), RequestPath = "/web", - ContentTypeProvider = extensionProvider + ContentTypeProvider = extensionProvider, + OnPrepareResponse = (context) => + { + if (Path.GetFileName(context.File.Name).Equals("index.html", StringComparison.Ordinal)) + { + context.Context.Response.Headers.CacheControl = new StringValues("no-cache"); + } + } }); mainApp.UseRobotsRedirection(); @@ -208,7 +217,6 @@ namespace Jellyfin.Server mainApp.UseRouting(); mainApp.UseAuthorization(); - mainApp.UseLanFiltering(); mainApp.UseIPBasedAccessValidation(); mainApp.UseWebSocketHandler(); mainApp.UseServerStartupMessage(); diff --git a/Jellyfin.Server/StartupOptions.cs b/Jellyfin.Server/StartupOptions.cs index 91ac827ca..4890ccbb2 100644 --- a/Jellyfin.Server/StartupOptions.cs +++ b/Jellyfin.Server/StartupOptions.cs @@ -74,6 +74,12 @@ namespace Jellyfin.Server public bool NoDetectNetworkChange { get; set; } /// <summary> + /// Gets or sets the path to an jellyfin backup archive to restore the application to. + /// </summary> + [Option("restore-archive", Required = false, HelpText = "Path to a Jellyfin backup archive to restore from")] + public string? RestoreArchive { get; set; } + + /// <summary> /// Gets the command line options as a dictionary that can be used in the .NET configuration system. /// </summary> /// <returns>The configuration dictionary.</returns> |