diff options
Diffstat (limited to 'Jellyfin.Server')
14 files changed, 1211 insertions, 357 deletions
diff --git a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs index c6c3f21fe..b04e55baa 100644 --- a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs +++ b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs @@ -119,15 +119,15 @@ namespace Jellyfin.Server.Extensions // https://github.com/dotnet/aspnetcore/blob/master/src/Middleware/HttpOverrides/src/ForwardedHeadersMiddleware.cs // Enable debug logging on Microsoft.AspNetCore.HttpOverrides.ForwardedHeadersMiddleware to help investigate issues. - options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost; - if (config.KnownProxies.Length == 0) { + options.ForwardedHeaders = ForwardedHeaders.None; options.KnownNetworks.Clear(); options.KnownProxies.Clear(); } else { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto | ForwardedHeaders.XForwardedHost; AddProxyAddresses(config, config.KnownProxies, options); } diff --git a/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs b/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs index 7695c0d9e..be9cf0f15 100644 --- a/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs +++ b/Jellyfin.Server/Extensions/WebHostBuilderExtensions.cs @@ -1,10 +1,14 @@ using System; +using System.Collections.Generic; using System.IO; using System.Net; +using System.Security.Cryptography.X509Certificates; using Jellyfin.Server.Helpers; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Extensions; +using MediaBrowser.Model.Net; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -35,56 +39,98 @@ public static class WebHostBuilderExtensions return builder .UseKestrel((builderContext, options) => { - var addresses = appHost.NetManager.GetAllBindInterfaces(false); + SetupJellyfinWebServer( + appHost.NetManager.GetAllBindInterfaces(false), + appHost.HttpPort, + appHost.ListenWithHttps ? appHost.HttpsPort : null, + appHost.Certificate, + startupConfig, + appPaths, + logger, + builderContext, + options); + }) + .UseStartup(context => new Startup(appHost, context.Configuration)); + } - bool flagged = false; - foreach (var netAdd in addresses) + /// <summary> + /// Configures a Kestrel type webServer to bind to the specific arguments. + /// </summary> + /// <param name="addresses">The IP addresses that should be listend to.</param> + /// <param name="httpPort">The http port.</param> + /// <param name="httpsPort">If set the https port. If set you must also set the certificate.</param> + /// <param name="certificate">The certificate used for https port.</param> + /// <param name="startupConfig">The startup config.</param> + /// <param name="appPaths">The app paths.</param> + /// <param name="logger">A logger.</param> + /// <param name="builderContext">The kestrel build pipeline context.</param> + /// <param name="options">The kestrel server options.</param> + /// <exception cref="InvalidOperationException">Will be thrown when a https port is set but no or an invalid certificate is provided.</exception> + public static void SetupJellyfinWebServer( + IReadOnlyList<IPData> addresses, + int httpPort, + int? httpsPort, + X509Certificate2? certificate, + IConfiguration startupConfig, + IApplicationPaths appPaths, + ILogger logger, + WebHostBuilderContext builderContext, + KestrelServerOptions options) + { + bool flagged = false; + foreach (var netAdd in addresses) + { + var address = netAdd.Address; + logger.LogInformation("Kestrel is listening on {Address}", address.Equals(IPAddress.IPv6Any) ? "all interfaces" : address); + options.Listen(netAdd.Address, httpPort); + if (httpsPort.HasValue) + { + if (builderContext.HostingEnvironment.IsDevelopment()) { - var address = netAdd.Address; - logger.LogInformation("Kestrel is listening on {Address}", address.Equals(IPAddress.IPv6Any) ? "all interfaces" : address); - options.Listen(netAdd.Address, appHost.HttpPort); - if (appHost.ListenWithHttps) + try { options.Listen( address, - appHost.HttpsPort, - listenOptions => listenOptions.UseHttps(appHost.Certificate)); + httpsPort.Value, + listenOptions => listenOptions.UseHttps()); } - else if (builderContext.HostingEnvironment.IsDevelopment()) + catch (InvalidOperationException) { - try + if (!flagged) { - options.Listen( - address, - appHost.HttpsPort, - listenOptions => listenOptions.UseHttps()); - } - catch (InvalidOperationException) - { - if (!flagged) - { - logger.LogWarning("Failed to listen to HTTPS using the ASP.NET Core HTTPS development certificate. Please ensure it has been installed and set as trusted"); - flagged = true; - } + logger.LogWarning("Failed to listen to HTTPS using the ASP.NET Core HTTPS development certificate. Please ensure it has been installed and set as trusted"); + flagged = true; } } } - - // Bind to unix socket (only on unix systems) - if (startupConfig.UseUnixSocket() && Environment.OSVersion.Platform == PlatformID.Unix) + else { - var socketPath = StartupHelpers.GetUnixSocketPath(startupConfig, appPaths); - - // Workaround for https://github.com/aspnet/AspNetCore/issues/14134 - if (File.Exists(socketPath)) + if (certificate is null) { - File.Delete(socketPath); + throw new InvalidOperationException("Cannot run jellyfin with https without setting a valid certificate."); } - options.ListenUnixSocket(socketPath); - logger.LogInformation("Kestrel listening to unix socket {SocketPath}", socketPath); + options.Listen( + address, + httpsPort.Value, + listenOptions => listenOptions.UseHttps(certificate)); } - }) - .UseStartup(context => new Startup(appHost, context.Configuration)); + } + } + + // Bind to unix socket (only on unix systems) + if (startupConfig.UseUnixSocket() && Environment.OSVersion.Platform == PlatformID.Unix) + { + var socketPath = StartupHelpers.GetUnixSocketPath(startupConfig, appPaths); + + // Workaround for https://github.com/aspnet/AspNetCore/issues/14134 + if (File.Exists(socketPath)) + { + File.Delete(socketPath); + } + + options.ListenUnixSocket(socketPath); + logger.LogInformation("Kestrel listening to unix socket {SocketPath}", socketPath); + } } } diff --git a/Jellyfin.Server/Filters/AdditionalModelFilter.cs b/Jellyfin.Server/Filters/AdditionalModelFilter.cs index bf38f741c..421eeecda 100644 --- a/Jellyfin.Server/Filters/AdditionalModelFilter.cs +++ b/Jellyfin.Server/Filters/AdditionalModelFilter.cs @@ -25,7 +25,7 @@ namespace Jellyfin.Server.Filters public class AdditionalModelFilter : IDocumentFilter { // Array of options that should not be visible in the api spec. - private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions) }; + private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions), typeof(MediaBrowser.Model.Branding.BrandingOptions) }; private readonly IServerConfigurationManager _serverConfigurationManager; /// <summary> @@ -92,17 +92,51 @@ namespace Jellyfin.Server.Filters continue; } - // Additional discriminator needed for GroupUpdate models... - if (messageType == SessionMessageType.SyncPlayGroupUpdate && type != typeof(SyncPlayGroupUpdateCommandMessage)) - { - continue; - } - var schema = context.SchemaGenerator.GenerateSchema(type, context.SchemaRepository); outboundWebSocketSchemas.Add(schema); outboundWebSocketDiscriminators.Add(messageType.ToString()!, schema.Reference.ReferenceV3); } + // Add custom "SyncPlayGroupUpdateMessage" schema because Swashbuckle cannot generate it for us + var syncPlayGroupUpdateMessageSchema = new OpenApiSchema + { + Type = "object", + Description = "Untyped sync play command.", + Properties = new Dictionary<string, OpenApiSchema> + { + { + "Data", new OpenApiSchema + { + AllOf = + [ + new OpenApiSchema { Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = nameof(GroupUpdate<object>) } } + ], + Description = "Group update data", + Nullable = false, + } + }, + { "MessageId", new OpenApiSchema { Type = "string", Format = "uuid", Description = "Gets or sets the message id." } }, + { + "MessageType", new OpenApiSchema + { + Enum = Enum.GetValues<SessionMessageType>().Select(type => new OpenApiString(type.ToString())).ToList<IOpenApiAny>(), + AllOf = + [ + new OpenApiSchema { Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = nameof(SessionMessageType) } } + ], + Description = "The different kinds of messages that are used in the WebSocket api.", + Default = new OpenApiString(nameof(SessionMessageType.SyncPlayGroupUpdate)), + ReadOnly = true + } + }, + }, + AdditionalPropertiesAllowed = false, + Reference = new OpenApiReference { Type = ReferenceType.Schema, Id = "SyncPlayGroupUpdateMessage" } + }; + context.SchemaRepository.AddDefinition("SyncPlayGroupUpdateMessage", syncPlayGroupUpdateMessageSchema); + outboundWebSocketSchemas.Add(syncPlayGroupUpdateMessageSchema); + outboundWebSocketDiscriminators[nameof(SessionMessageType.SyncPlayGroupUpdate)] = syncPlayGroupUpdateMessageSchema.Reference.ReferenceV3; + var outboundWebSocketMessageSchema = new OpenApiSchema { Type = "object", @@ -140,41 +174,46 @@ namespace Jellyfin.Server.Filters }); // Manually generate sync play GroupUpdate messages. - if (!context.SchemaRepository.Schemas.TryGetValue(nameof(GroupUpdate), out var groupUpdateSchema)) - { - groupUpdateSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate), context.SchemaRepository); - } - - var groupUpdateOfGroupInfoSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<GroupInfoDto>), context.SchemaRepository); - var groupUpdateOfGroupStateSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<GroupStateUpdate>), context.SchemaRepository); - var groupUpdateOfStringSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<string>), context.SchemaRepository); - var groupUpdateOfPlayQueueSchema = context.SchemaGenerator.GenerateSchema(typeof(GroupUpdate<PlayQueueUpdate>), context.SchemaRepository); + var groupUpdateTypes = typeof(GroupUpdate<>).Assembly.GetTypes() + .Where(t => t.BaseType != null + && t.BaseType.IsGenericType + && t.BaseType.GetGenericTypeDefinition() == typeof(GroupUpdate<>)) + .ToList(); - groupUpdateSchema.OneOf = new List<OpenApiSchema> + var groupUpdateSchemas = new List<OpenApiSchema>(); + var groupUpdateDiscriminators = new Dictionary<string, string>(); + foreach (var type in groupUpdateTypes) { - groupUpdateOfGroupInfoSchema, - groupUpdateOfGroupStateSchema, - groupUpdateOfStringSchema, - groupUpdateOfPlayQueueSchema - }; + var groupUpdateType = (GroupUpdateType?)type.GetProperty(nameof(GroupUpdate<object>.Type))?.GetCustomAttribute<DefaultValueAttribute>()?.Value; + if (groupUpdateType is null) + { + continue; + } + + var schema = context.SchemaGenerator.GenerateSchema(type, context.SchemaRepository); + groupUpdateSchemas.Add(schema); + groupUpdateDiscriminators[groupUpdateType.ToString()!] = schema.Reference.ReferenceV3; + } - groupUpdateSchema.Discriminator = new OpenApiDiscriminator + var groupUpdateSchema = new OpenApiSchema { - PropertyName = nameof(GroupUpdate.Type), - Mapping = new Dictionary<string, string> + Type = "object", + Description = "Represents the list of possible group update types", + Reference = new OpenApiReference + { + Id = nameof(GroupUpdate<object>), + Type = ReferenceType.Schema + }, + OneOf = groupUpdateSchemas, + Discriminator = new OpenApiDiscriminator { - { GroupUpdateType.UserJoined.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.UserLeft.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupJoined.ToString(), groupUpdateOfGroupInfoSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupLeft.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.StateUpdate.ToString(), groupUpdateOfGroupStateSchema.Reference.ReferenceV3 }, - { GroupUpdateType.PlayQueue.ToString(), groupUpdateOfPlayQueueSchema.Reference.ReferenceV3 }, - { GroupUpdateType.NotInGroup.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.GroupDoesNotExist.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 }, - { GroupUpdateType.LibraryAccessDenied.ToString(), groupUpdateOfStringSchema.Reference.ReferenceV3 } + PropertyName = nameof(GroupUpdate<object>.Type), + Mapping = groupUpdateDiscriminators } }; + context.SchemaRepository.Schemas[nameof(GroupUpdate<object>)] = groupUpdateSchema; + context.SchemaGenerator.GenerateSchema(typeof(ServerDiscoveryInfo), context.SchemaRepository); foreach (var configuration in _serverConfigurationManager.GetConfigurationStores()) diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs index fd540c9c0..c223576da 100644 --- a/Jellyfin.Server/Migrations/MigrationRunner.cs +++ b/Jellyfin.Server/Migrations/MigrationRunner.cs @@ -29,7 +29,8 @@ namespace Jellyfin.Server.Migrations typeof(PreStartupRoutines.CreateNetworkConfiguration), typeof(PreStartupRoutines.MigrateMusicBrainzTimeout), typeof(PreStartupRoutines.MigrateNetworkConfiguration), - typeof(PreStartupRoutines.MigrateEncodingOptions) + typeof(PreStartupRoutines.MigrateEncodingOptions), + typeof(PreStartupRoutines.RenameEnableGroupingIntoCollections) }; /// <summary> @@ -48,13 +49,15 @@ namespace Jellyfin.Server.Migrations typeof(Routines.RemoveDownloadImagesInAdvance), typeof(Routines.MigrateAuthenticationDb), typeof(Routines.FixPlaylistOwner), - typeof(Routines.MigrateRatingLevels), typeof(Routines.AddDefaultCastReceivers), typeof(Routines.UpdateDefaultPluginRepository), typeof(Routines.FixAudioData), - typeof(Routines.MoveTrickplayFiles), typeof(Routines.RemoveDuplicatePlaylistChildren), typeof(Routines.MigrateLibraryDb), + typeof(Routines.MoveExtractedFiles), + typeof(Routines.MigrateRatingLevels), + typeof(Routines.MoveTrickplayFiles), + typeof(Routines.MigrateKeyframeData), }; /// <summary> @@ -146,11 +149,18 @@ namespace Jellyfin.Server.Migrations } } + List<IMigrationRoutine> databaseMigrations = []; try { foreach (var migrationRoutine in migrationsToBeApplied) { logger.LogInformation("Applying migration '{Name}'", migrationRoutine.Name); + var isDbMigration = migrationRoutine is IDatabaseMigrationRoutine; + + if (isDbMigration) + { + databaseMigrations.Add(migrationRoutine); + } try { @@ -164,17 +174,31 @@ namespace Jellyfin.Server.Migrations // Mark the migration as completed logger.LogInformation("Migration '{Name}' applied successfully", migrationRoutine.Name); - migrationOptions.Applied.Add((migrationRoutine.Id, migrationRoutine.Name)); - saveConfiguration(migrationOptions); - logger.LogDebug("Migration '{Name}' marked as applied in configuration.", migrationRoutine.Name); + if (!isDbMigration) + { + migrationOptions.Applied.Add((migrationRoutine.Id, migrationRoutine.Name)); + saveConfiguration(migrationOptions); + logger.LogDebug("Migration '{Name}' marked as applied in configuration.", migrationRoutine.Name); + } } } - catch (System.Exception) when (migrationKey is not null && jellyfinDatabaseProvider is not null) + catch (Exception) when (migrationKey is not null && jellyfinDatabaseProvider is not null) { - logger.LogInformation("Rollback on database as migration reported failure."); - await jellyfinDatabaseProvider.RestoreBackupFast(migrationKey, CancellationToken.None).ConfigureAwait(false); + if (databaseMigrations.Count != 0) + { + logger.LogInformation("Rolling back database as migrations reported failure."); + await jellyfinDatabaseProvider.RestoreBackupFast(migrationKey, CancellationToken.None).ConfigureAwait(false); + } + throw; } + + foreach (var migrationRoutine in databaseMigrations) + { + migrationOptions.Applied.Add((migrationRoutine.Id, migrationRoutine.Name)); + saveConfiguration(migrationOptions); + logger.LogDebug("Migration '{Name}' marked as applied in configuration.", migrationRoutine.Name); + } } } } diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs new file mode 100644 index 000000000..0a37b35a6 --- /dev/null +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; +using Emby.Server.Implementations; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.PreStartupRoutines; + +/// <inheritdoc /> +public class RenameEnableGroupingIntoCollections : IMigrationRoutine +{ + private readonly ServerApplicationPaths _applicationPaths; + private readonly ILogger<RenameEnableGroupingIntoCollections> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="RenameEnableGroupingIntoCollections"/> class. + /// </summary> + /// <param name="applicationPaths">An instance of <see cref="ServerApplicationPaths"/>.</param> + /// <param name="loggerFactory">An instance of the <see cref="ILoggerFactory"/> interface.</param> + public RenameEnableGroupingIntoCollections(ServerApplicationPaths applicationPaths, ILoggerFactory loggerFactory) + { + _applicationPaths = applicationPaths; + _logger = loggerFactory.CreateLogger<RenameEnableGroupingIntoCollections>(); + } + + /// <inheritdoc /> + public Guid Id => Guid.Parse("E73B777D-CD5C-4E71-957A-B86B3660B7CF"); + + /// <inheritdoc /> + public string Name => nameof(RenameEnableGroupingIntoCollections); + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "system.xml"); + if (!File.Exists(path)) + { + _logger.LogWarning("Configuration file not found: {Path}", path); + return; + } + + try + { + XDocument xmlDocument = XDocument.Load(path); + var element = xmlDocument.Descendants("EnableGroupingIntoCollections").FirstOrDefault(); + if (element is not null) + { + element.Name = "EnableGroupingMoviesIntoCollections"; + _logger.LogInformation("The tag <EnableGroupingIntoCollections> was successfully renamed to <EnableGroupingMoviesIntoCollections>."); + xmlDocument.Save(path); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "An error occurred while updating the XML file: {Message}", ex.Message); + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs new file mode 100644 index 000000000..b8e69db8e --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +public class MigrateKeyframeData : IDatabaseMigrationRoutine +{ + private readonly ILibraryManager _libraryManager; + private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IApplicationPaths _appPaths; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. + /// </summary> + /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="dbProvider">The EFCore db factory.</param> + public MigrateKeyframeData( + ILibraryManager libraryManager, + ILogger<MoveTrickplayFiles> logger, + IApplicationPaths appPaths, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _libraryManager = libraryManager; + _logger = logger; + _appPaths = appPaths; + _dbProvider = dbProvider; + } + + private string KeyframeCachePath => Path.Combine(_appPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public Guid Id => new("EA4bCAE1-09A4-428E-9B90-4B4FD2EA1B24"); + + /// <inheritdoc /> + public string Name => "MigrateKeyframeData"; + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 100; + int itemCount = 0, offset = 0, previousCount; + + var sw = Stopwatch.StartNew(); + var itemsQuery = new InternalItemsQuery + { + MediaTypes = [MediaType.Video], + SourceTypes = [SourceType.Library], + IsVirtualItem = false, + IsFolder = false + }; + + using var context = _dbProvider.CreateDbContext(); + context.KeyframeData.ExecuteDelete(); + using var transaction = context.Database.BeginTransaction(); + List<KeyframeData> keyframes = []; + + do + { + var result = _libraryManager.GetItemsResult(itemsQuery); + _logger.LogInformation("Importing keyframes for {Count} items", result.TotalRecordCount); + + var items = result.Items; + previousCount = items.Count; + offset += Limit; + foreach (var item in items) + { + if (TryGetKeyframeData(item, out var data)) + { + keyframes.Add(data); + } + + if (++itemCount % 10_000 == 0) + { + context.KeyframeData.AddRange(keyframes); + keyframes.Clear(); + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + } + } + } while (previousCount == Limit); + + context.KeyframeData.AddRange(keyframes); + context.SaveChanges(); + transaction.Commit(); + + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + + if (Directory.Exists(KeyframeCachePath)) + { + Directory.Delete(KeyframeCachePath, true); + } + } + + private bool TryGetKeyframeData(BaseItem item, [NotNullWhen(true)] out KeyframeData? data) + { + data = null; + var path = item.Path; + if (!string.IsNullOrEmpty(path)) + { + var cachePath = GetCachePath(KeyframeCachePath, path); + if (TryReadFromCache(cachePath, out var keyframeData)) + { + data = new() + { + ItemId = item.Id, + KeyframeTicks = keyframeData.KeyframeTicks.ToList(), + TotalDuration = keyframeData.TotalDuration + }; + + return true; + } + } + + return false; + } + + private string? GetCachePath(string keyframeCachePath, string filePath) + { + DateTime? lastWriteTimeUtc; + try + { + lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping {Path}: {Exception}", filePath, e.Message); + + return null; + } + + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string? cachePath, [NotNullWhen(true)] out MediaEncoding.Keyframes.KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<MediaEncoding.Keyframes.KeyframeData>(bytes, _jsonOptions); + + return cachedResult is not null; + } + + cachedResult = null; + + return false; + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index 8e462015f..105fd555f 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -73,273 +73,352 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine var dataPath = _paths.DataPath; var libraryDbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={libraryDbPath}"); - var migrationTotalTime = TimeSpan.Zero; + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); - var stopwatch = new Stopwatch(); - stopwatch.Start(); + var fullOperationTimer = new Stopwatch(); + fullOperationTimer.Start(); - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving TypedBaseItem."); - const string typedBaseItemsQuery = """ - SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, - IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, - PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, - ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, - Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, - DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, - PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, - ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName FROM TypedBaseItems - """; - dbContext.BaseItems.ExecuteDelete(); + using (var operation = GetPreparedDbContext("Cleanup database")) + { + operation.JellyfinDbContext.AttachmentStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.BaseItems.ExecuteDelete(); + operation.JellyfinDbContext.ItemValues.ExecuteDelete(); + operation.JellyfinDbContext.UserData.ExecuteDelete(); + operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.Peoples.ExecuteDelete(); + operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete(); + operation.JellyfinDbContext.Chapters.ExecuteDelete(); + operation.JellyfinDbContext.AncestorIds.ExecuteDelete(); + } var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); - foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) - { - var baseItem = GetItem(dto); - dbContext.BaseItems.Add(baseItem.BaseItem); - foreach (var dataKey in baseItem.LegacyUserDataKey) + connection.Open(); + + var baseItemIds = new HashSet<Guid>(); + using (var operation = GetPreparedDbContext("moving TypedBaseItem")) + { + const string typedBaseItemsQuery = + """ + SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, + IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, + PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, + ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, + Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, + DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, + PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, + ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems + """; + using (new TrackedMigrationStep("Loading TypedBaseItems", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + { + var baseItem = GetItem(dto); + operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem); + baseItemIds.Add(baseItem.BaseItem.Id); + foreach (var dataKey in baseItem.LegacyUserDataKey) + { + legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + } + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger)) { - legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + operation.JellyfinDbContext.SaveChanges(); } } - _logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (var operation = GetPreparedDbContext("moving ItemValues")) + { + // do not migrate inherited types as they are now properly mapped in search and lookup. + const string itemValueQuery = + """ + SELECT ItemId, Type, Value, CleanValue FROM ItemValues + WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) + """; - _logger.LogInformation("Start moving ItemValues."); - // do not migrate inherited types as they are now properly mapped in search and lookup. - const string itemValueQuery = - """ - SELECT ItemId, Type, Value, CleanValue FROM ItemValues - WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) - """; - dbContext.ItemValues.ExecuteDelete(); + // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. + var localItems = new Dictionary<(int Type, string Value), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep("loading ItemValues", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + { + var itemId = dto.GetGuid(0); + var entity = GetItemValue(dto); + var key = ((int)entity.Type, entity.Value); + if (!localItems.TryGetValue(key, out var existing)) + { + localItems[key] = existing = (entity, []); + } - // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. - var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + existing.ItemIds.Add(itemId); + } - foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) - { - var itemId = dto.GetGuid(0); - var entity = GetItemValue(dto); - var key = ((int)entity.Type, entity.CleanValue); - if (!localItems.TryGetValue(key, out var existing)) - { - localItems[key] = existing = (entity, []); + foreach (var item in localItems) + { + operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue); + operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + { + Item = null!, + ItemValue = null!, + ItemId = f, + ItemValueId = item.Value.ItemValue.ItemValueId + })); + } } - existing.ItemIds.Add(itemId); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - foreach (var item in localItems) + using (var operation = GetPreparedDbContext("moving UserData")) { - dbContext.ItemValues.Add(item.Value.ItemValue); - dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + var queryResult = connection.Query( + """ + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) + """); + + using (new TrackedMigrationStep("loading UserData", _logger)) { - Item = null!, - ItemValue = null!, - ItemId = f, - ItemValueId = item.Value.ItemValue.ItemValueId - })); - } + var users = operation.JellyfinDbContext.Users.AsNoTracking().ToImmutableArray(); + var userIdBlacklist = new HashSet<int>(); - _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + foreach (var entity in queryResult) + { + var userData = GetUserData(users, entity, userIdBlacklist); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); - _logger.LogInformation("Start moving UserData."); - var queryResult = connection.Query(""" - SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) - """); + continue; + } - dbContext.UserData.ExecuteDelete(); + if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + { + _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); + continue; + } - var users = dbContext.Users.AsNoTracking().ToImmutableArray(); + userData.ItemId = refItem.Id; + operation.JellyfinDbContext.UserData.Add(userData); + } - foreach (var entity in queryResult) - { - var userData = GetUserData(users, entity); - if (userData is null) - { - _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0)); - continue; + users.Clear(); } - if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + legacyBaseItemWithUserKeys.Clear(); + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger)) { - _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); - continue; + operation.JellyfinDbContext.SaveChanges(); } - - userData.ItemId = refItem.Id; - dbContext.UserData.Add(userData); } - users.Clear(); - legacyBaseItemWithUserKeys.Clear(); - _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count); - dbContext.SaveChanges(); - - _logger.LogInformation("Start moving MediaStreamInfos."); - const string mediaStreamQuery = """ - SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, - IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, - AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, - Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, - DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired - FROM MediaStreams - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) - """; - dbContext.MediaStreamInfos.ExecuteDelete(); - - foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + using (var operation = GetPreparedDbContext("moving MediaStreamInfos")) { - dbContext.MediaStreamInfos.Add(GetMediaStream(dto)); - } - - _logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); + const string mediaStreamQuery = + """ + SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, + IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, + AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, + Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, + DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired + FROM MediaStreams + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) + """; - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving People."); - const string personsQuery = """ - SELECT ItemId, Name, Role, PersonType, SortOrder FROM People - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) - """; - dbContext.Peoples.ExecuteDelete(); - dbContext.PeopleBaseItemMap.ExecuteDelete(); - - var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - var baseItemIds = dbContext.BaseItems.Select(b => b.Id).ToHashSet(); + using (new TrackedMigrationStep("loading MediaStreamInfos", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + { + operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto)); + } + } - foreach (SqliteDataReader reader in connection.Query(personsQuery)) - { - var itemId = reader.GetGuid(0); - if (!baseItemIds.Contains(itemId)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger)) { - _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); - continue; + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving AttachmentStreamInfos")) + { + const string mediaAttachmentQuery = + """ + SELECT ItemId, AttachmentIndex, Codec, CodecTag, Comment, filename, MIMEType + FROM mediaattachments + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = mediaattachments.ItemId) + """; - var entity = GetPerson(reader); - if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + using (new TrackedMigrationStep("loading AttachmentStreamInfos", _logger)) { - peopleCache[entity.Name] = personCache = (entity, []); + foreach (SqliteDataReader dto in connection.Query(mediaAttachmentQuery)) + { + operation.JellyfinDbContext.AttachmentStreamInfos.Add(GetMediaAttachment(dto)); + } } - if (reader.TryGetString(2, out var role)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AttachmentStreamInfos.Local.Count} AttachmentStreamInfos entries", _logger)) { + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving People")) + { + const string personsQuery = + """ + SELECT ItemId, Name, Role, PersonType, SortOrder FROM People + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) + """; - int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); + var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - personCache.Items.Add(new PeopleBaseItemMap() + using (new TrackedMigrationStep("loading People", _logger)) { - Item = null!, - ItemId = itemId, - People = null!, - PeopleId = personCache.Person.Id, - ListOrder = sortOrder, - SortOrder = sortOrder, - Role = role - }); - } + foreach (SqliteDataReader reader in connection.Query(personsQuery)) + { + var itemId = reader.GetGuid(0); + if (!baseItemIds.Contains(itemId)) + { + _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); + continue; + } + + var entity = GetPerson(reader); + if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + { + peopleCache[entity.Name] = personCache = (entity, []); + } - baseItemIds.Clear(); + if (reader.TryGetString(2, out var role)) + { + } - foreach (var item in peopleCache) - { - dbContext.Peoples.Add(item.Value.Person); - dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); - } + int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); - peopleCache.Clear(); + personCache.Items.Add(new PeopleBaseItemMap() + { + Item = null!, + ItemId = itemId, + People = null!, + PeopleId = personCache.Person.Id, + ListOrder = sortOrder, + SortOrder = sortOrder, + Role = role + }); + } - _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + baseItemIds.Clear(); - _logger.LogInformation("Start moving Chapters."); - const string chapterQuery = """ - SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) - """; - dbContext.Chapters.ExecuteDelete(); + foreach (var item in peopleCache) + { + operation.JellyfinDbContext.Peoples.Add(item.Value.Person); + operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); + } - foreach (SqliteDataReader dto in connection.Query(chapterQuery)) - { - var chapter = GetChapter(dto); - dbContext.Chapters.Add(chapter); + peopleCache.Clear(); + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (var operation = GetPreparedDbContext("moving Chapters")) + { + const string chapterQuery = + """ + SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) + """; - _logger.LogInformation("Start moving AncestorIds."); - const string ancestorIdsQuery = """ - SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds - WHERE - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) - AND - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) - """; - dbContext.Chapters.ExecuteDelete(); + using (new TrackedMigrationStep("loading Chapters", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(chapterQuery)) + { + var chapter = GetChapter(dto); + operation.JellyfinDbContext.Chapters.Add(chapter); + } + } - foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) - { - var ancestorId = GetAncestorId(dto); - dbContext.AncestorIds.Add(ancestorId); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.Chapters.Local.Count); + using (var operation = GetPreparedDbContext("moving AncestorIds")) + { + const string ancestorIdsQuery = + """ + SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds + WHERE + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) + AND + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) + """; + + using (new TrackedMigrationStep("loading AncestorIds", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + { + var ancestorId = GetAncestorId(dto); + operation.JellyfinDbContext.AncestorIds.Add(ancestorId); + } + } - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } connection.Close(); + _logger.LogInformation("Migration of the Library.db done."); - _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + _logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed); SqliteConnection.ClearAllPools(); + _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); File.Move(libraryDbPath, libraryDbPath + ".old", true); - _logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime); - _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); } - private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto) + private DatabaseMigrationStep GetPreparedDbContext(string operationName) + { + var dbContext = _provider.CreateDbContext(); + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return new DatabaseMigrationStep(dbContext, operationName, _logger); + } + + private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto, HashSet<int> userIdBlacklist) { var internalUserId = dto.GetInt32(1); var user = users.FirstOrDefault(e => e.InternalId == internalUserId); if (user is null) { + if (userIdBlacklist.Contains(internalUserId)) + { + return null; + } + _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); return null; } @@ -654,6 +733,48 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return item; } + /// <summary> + /// Gets the attachment. + /// </summary> + /// <param name="reader">The reader.</param> + /// <returns>MediaAttachment.</returns> + private AttachmentStreamInfo GetMediaAttachment(SqliteDataReader reader) + { + var item = new AttachmentStreamInfo + { + Index = reader.GetInt32(1), + Item = null!, + ItemId = reader.GetGuid(0), + }; + + if (reader.TryGetString(2, out var codec)) + { + item.Codec = codec; + } + + if (reader.TryGetString(3, out var codecTag)) + { + item.CodecTag = codecTag; + } + + if (reader.TryGetString(4, out var comment)) + { + item.Comment = comment; + } + + if (reader.TryGetString(5, out var fileName)) + { + item.Filename = fileName; + } + + if (reader.TryGetString(6, out var mimeType)) + { + item.MimeType = mimeType; + } + + return item; + } + private (BaseItemEntity BaseItem, string[] LegacyUserDataKey) GetItem(SqliteDataReader reader) { var entity = new BaseItemEntity() @@ -1045,6 +1166,11 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine entity.CleanName = cleanName; } + if (reader.TryGetString(index++, out var unratedType)) + { + entity.UnratedType = unratedType; + } + var baseItem = BaseItemRepository.DeserialiseBaseItem(entity, _logger, null, false); var dataKeys = baseItem.GetUserDataKeys(); userDataKeys.AddRange(dataKeys); @@ -1209,4 +1335,58 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return image; } + + private class TrackedMigrationStep : IDisposable + { + private readonly string _operationName; + private readonly ILogger _logger; + private readonly Stopwatch _operationTimer; + private bool _disposed; + + public TrackedMigrationStep(string operationName, ILogger logger) + { + _operationName = operationName; + _logger = logger; + _operationTimer = Stopwatch.StartNew(); + logger.LogInformation("Start {OperationName}", operationName); + } + + public bool Disposed + { + get => _disposed; + set => _disposed = value; + } + + public virtual void Dispose() + { + if (Disposed) + { + return; + } + + Disposed = true; + _logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed); + } + } + + private sealed class DatabaseMigrationStep : TrackedMigrationStep + { + public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger) + { + JellyfinDbContext = jellyfinDbContext; + } + + public JellyfinDbContext JellyfinDbContext { get; } + + public override void Dispose() + { + if (Disposed) + { + return; + } + + JellyfinDbContext.Dispose(); + base.Dispose(); + } + } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 9c2184029..c38beb723 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,36 +1,33 @@ using System; -using System.Globalization; -using System.IO; -using Emby.Server.Implementations.Data; -using MediaBrowser.Controller; +using System.Linq; +using Jellyfin.Database.Implementations; using MediaBrowser.Model.Globalization; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines { /// <summary> - /// Migrate rating levels to new rating level system. + /// Migrate rating levels. /// </summary> - internal class MigrateRatingLevels : IMigrationRoutine + internal class MigrateRatingLevels : IDatabaseMigrationRoutine { - private const string DbFilename = "library.db"; private readonly ILogger<MigrateRatingLevels> _logger; - private readonly IServerApplicationPaths _applicationPaths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; private readonly ILocalizationManager _localizationManager; public MigrateRatingLevels( - IServerApplicationPaths applicationPaths, + IDbContextFactory<JellyfinDbContext> provider, ILoggerFactory loggerFactory, ILocalizationManager localizationManager) { - _applicationPaths = applicationPaths; + _provider = provider; _localizationManager = localizationManager; _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); } /// <inheritdoc/> - public Guid Id => Guid.Parse("{73DAB92A-178B-48CD-B05B-FE18733ACDC8}"); + public Guid Id => Guid.Parse("{98724538-EB11-40E3-931A-252C55BDDE7A}"); /// <inheritdoc/> public string Name => "MigrateRatingLevels"; @@ -41,54 +38,37 @@ namespace Jellyfin.Server.Migrations.Routines /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) + _logger.LogInformation("Recalculating parental rating levels based on rating string."); + using var context = _provider.CreateDbContext(); + using var transaction = context.Database.BeginTransaction(); + var ratings = context.BaseItems.AsNoTracking().Select(e => e.OfficialRating).Distinct(); + foreach (var rating in ratings) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + if (string.IsNullOrEmpty(rating)) { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } + int? value = null; + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, value)); + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, value)); } - } - - // Migrate parental rating strings to new levels - _logger.LogInformation("Recalculating parental rating levels based on rating string."); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) - { - var queryResult = connection.Query("SELECT DISTINCT OfficialRating FROM TypedBaseItems"); - foreach (var entry in queryResult) + else { - if (!entry.TryGetString(0, out var ratingString) || string.IsNullOrEmpty(ratingString)) - { - connection.Execute("UPDATE TypedBaseItems SET InheritedParentalRatingValue = NULL WHERE OfficialRating IS NULL OR OfficialRating='';"); - } - else - { - var ratingValue = _localizationManager.GetRatingLevel(ratingString)?.ToString(CultureInfo.InvariantCulture) ?? "NULL"; - - using var statement = connection.PrepareStatement("UPDATE TypedBaseItems SET InheritedParentalRatingValue = @Value WHERE OfficialRating = @Rating;"); - statement.TryBind("@Value", ratingValue); - statement.TryBind("@Rating", ratingString); - statement.ExecuteNonQuery(); - } + var ratingValue = _localizationManager.GetRatingScore(rating); + var score = ratingValue?.Score; + var subScore = ratingValue?.SubScore; + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, score)); + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, subScore)); } - - transaction.Commit(); } + + transaction.Commit(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs index c40560660..1b5fab7a8 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs @@ -112,7 +112,8 @@ namespace Jellyfin.Server.Migrations.Routines { Id = entry.GetGuid(1), InternalId = entry.GetInt64(0), - MaxParentalAgeRating = policy.MaxParentalRating, + MaxParentalRatingScore = policy.MaxParentalRating, + MaxParentalRatingSubScore = null, EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs new file mode 100644 index 000000000..f63c5fd40 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -0,0 +1,299 @@ +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Jellyfin.Data.Enums; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.MediaInfo; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +public class MoveExtractedFiles : IDatabaseMigrationRoutine +{ + private readonly IApplicationPaths _appPaths; + private readonly ILibraryManager _libraryManager; + private readonly ILogger<MoveExtractedFiles> _logger; + private readonly IMediaSourceManager _mediaSourceManager; + private readonly IPathManager _pathManager; + + /// <summary> + /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class. + /// </summary> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="mediaSourceManager">Instance of the <see cref="IMediaSourceManager"/> interface.</param> + /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> + public MoveExtractedFiles( + IApplicationPaths appPaths, + ILibraryManager libraryManager, + ILogger<MoveExtractedFiles> logger, + IMediaSourceManager mediaSourceManager, + IPathManager pathManager) + { + _appPaths = appPaths; + _libraryManager = libraryManager; + _logger = logger; + _mediaSourceManager = mediaSourceManager; + _pathManager = pathManager; + } + + private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles"); + + private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); + + /// <inheritdoc /> + public Guid Id => new("9063b0Ef-CFF1-4EDC-9A13-74093681A89B"); + + /// <inheritdoc /> + public string Name => "MoveExtractedFiles"; + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 500; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + var itemsQuery = new InternalItemsQuery + { + MediaTypes = [MediaType.Video], + SourceTypes = [SourceType.Library], + IsVirtualItem = false, + IsFolder = false, + Limit = Limit, + StartIndex = offset, + EnableTotalRecordCount = true, + }; + + var records = _libraryManager.GetItemsResult(itemsQuery).TotalRecordCount; + _logger.LogInformation("Checking {Count} items for movable extracted files.", records); + + // Make sure directories exist + Directory.CreateDirectory(SubtitleCachePath); + Directory.CreateDirectory(AttachmentCachePath); + + itemsQuery.EnableTotalRecordCount = false; + do + { + itemsQuery.StartIndex = offset; + var result = _libraryManager.GetItemsResult(itemsQuery); + + var items = result.Items; + foreach (var item in items) + { + if (MoveSubtitleAndAttachmentFiles(item)) + { + itemCount++; + } + } + + offset += Limit; + if (offset % 5_000 == 0) + { + _logger.LogInformation("Checked extracted files for {Count} items in {Time}.", offset, sw.Elapsed); + } + } while (offset < records); + + _logger.LogInformation("Checked {Checked} items - Moved files for {Items} items in {Time}.", records, itemCount, sw.Elapsed); + + // Get all subdirectories with 1 character names (those are the legacy directories) + var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList(); + subdirectories.AddRange(Directory.GetDirectories(AttachmentCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == AttachmentCachePath.Length + 2)); + + // Remove all legacy subdirectories + foreach (var subdir in subdirectories) + { + Directory.Delete(subdir, true); + } + + // Remove old cache path + var attachmentCachePath = Path.Join(_appPaths.CachePath, "attachments"); + if (Directory.Exists(attachmentCachePath)) + { + Directory.Delete(attachmentCachePath, true); + } + + _logger.LogInformation("Cleaned up left over subtitles and attachments."); + } + + private bool MoveSubtitleAndAttachmentFiles(BaseItem item) + { + var mediaStreams = item.GetMediaStreams().Where(s => s.Type == MediaStreamType.Subtitle && !s.IsExternal); + var itemIdString = item.Id.ToString("N", CultureInfo.InvariantCulture); + var modified = false; + foreach (var mediaStream in mediaStreams) + { + if (mediaStream.Codec is null) + { + continue; + } + + var mediaStreamIndex = mediaStream.Index; + var extension = GetSubtitleExtension(mediaStream.Codec); + var oldSubtitleCachePath = GetOldSubtitleCachePath(item.Path, mediaStream.Index, extension); + if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath)) + { + continue; + } + + var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension); + if (File.Exists(newSubtitleCachePath)) + { + File.Delete(oldSubtitleCachePath); + } + else + { + var newDirectory = Path.GetDirectoryName(newSubtitleCachePath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldSubtitleCachePath, newSubtitleCachePath, false); + _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, item.Id, oldSubtitleCachePath, newSubtitleCachePath); + + modified = true; + } + } + } + + var attachments = _mediaSourceManager.GetMediaAttachments(item.Id).Where(a => !string.Equals(a.Codec, "mjpeg", StringComparison.OrdinalIgnoreCase)).ToList(); + var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.FileName) + && (a.FileName.Contains('/', StringComparison.OrdinalIgnoreCase) || a.FileName.Contains('\\', StringComparison.OrdinalIgnoreCase))); + foreach (var attachment in attachments) + { + var attachmentIndex = attachment.Index; + var oldAttachmentPath = GetOldAttachmentDataPath(item.Path, attachmentIndex); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + continue; + } + } + + var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.FileName ?? attachmentIndex.ToString(CultureInfo.InvariantCulture)); + if (File.Exists(newAttachmentPath)) + { + File.Delete(oldAttachmentPath); + } + else + { + var newDirectory = Path.GetDirectoryName(newAttachmentPath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldAttachmentPath, newAttachmentPath, false); + _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, item.Id, oldAttachmentPath, newAttachmentPath); + + modified = true; + } + } + } + + return modified; + } + + private string? GetOldAttachmentDataPath(string? mediaPath, int attachmentStreamIndex) + { + if (mediaPath is null) + { + return null; + } + + string filename; + var protocol = _mediaSourceManager.GetPathProtocol(mediaPath); + if (protocol == MediaProtocol.File) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(mediaPath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping attachment at index {Index} for {Path}: {Exception}", attachmentStreamIndex, mediaPath, e.Message); + + return null; + } + + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + else + { + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + + return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename); + } + + private string? GetOldAttachmentCachePath(string mediaSourceId, MediaAttachment attachment, bool shouldExtractOneByOne) + { + var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId); + if (shouldExtractOneByOne) + { + return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture)); + } + + if (string.IsNullOrEmpty(attachment.FileName)) + { + return null; + } + + return Path.Join(attachmentFolderPath, attachment.FileName); + } + + private string? GetOldSubtitleCachePath(string path, int streamIndex, string outputSubtitleExtension) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(path); + } + catch (IOException e) + { + _logger.LogDebug("Skipping subtitle at index {Index} for {Path}: {Exception}", streamIndex, path, e.Message); + + return null; + } + + var ticksParam = string.Empty; + ReadOnlySpan<char> filename = new Guid(MD5.HashData(Encoding.Unicode.GetBytes(path + "_" + streamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture) + ticksParam))) + outputSubtitleExtension; + + return Path.Join(SubtitleCachePath, filename[..1], filename); + } + + private static string GetSubtitleExtension(string codec) + { + if (codec.ToLower(CultureInfo.InvariantCulture).Equals("ass", StringComparison.OrdinalIgnoreCase) + || codec.ToLower(CultureInfo.InvariantCulture).Equals("ssa", StringComparison.OrdinalIgnoreCase)) + { + return "." + codec; + } + else if (codec.Contains("pgs", StringComparison.OrdinalIgnoreCase)) + { + return ".sup"; + } + else + { + return ".srt"; + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index f4ebac377..eeb11e14c 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,7 +4,6 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; -using MediaBrowser.Common; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -16,7 +15,7 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to move trickplay files to the new directory. /// </summary> -public class MoveTrickplayFiles : IMigrationRoutine +public class MoveTrickplayFiles : IDatabaseMigrationRoutine { private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs index f84bccc25..e183a1d63 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs @@ -1,12 +1,10 @@ using System; using System.Linq; using System.Threading; - using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Playlists; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines; @@ -15,16 +13,13 @@ namespace Jellyfin.Server.Migrations.Routines; /// </summary> internal class RemoveDuplicatePlaylistChildren : IMigrationRoutine { - private readonly ILogger<RemoveDuplicatePlaylistChildren> _logger; private readonly ILibraryManager _libraryManager; private readonly IPlaylistManager _playlistManager; public RemoveDuplicatePlaylistChildren( - ILogger<RemoveDuplicatePlaylistChildren> logger, ILibraryManager libraryManager, IPlaylistManager playlistManager) { - _logger = logger; _libraryManager = libraryManager; _playlistManager = playlistManager; } diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs index e661d0d4a..8d0bf73f6 100644 --- a/Jellyfin.Server/Program.cs +++ b/Jellyfin.Server/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -11,6 +12,7 @@ using Emby.Server.Implementations; using Jellyfin.Database.Implementations; using Jellyfin.Server.Extensions; using Jellyfin.Server.Helpers; +using Jellyfin.Server.Implementations.StorageHelpers; using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; @@ -46,7 +48,7 @@ namespace Jellyfin.Server public const string LoggingConfigFileSystem = "logging.json"; private static readonly SerilogLoggerFactory _loggerFactory = new SerilogLoggerFactory(); - private static SetupServer _setupServer = new(); + private static SetupServer? _setupServer; private static CoreAppHost? _appHost; private static IHost? _jellyfinHost = null; private static long _startTimestamp; @@ -75,7 +77,6 @@ namespace Jellyfin.Server { _startTimestamp = Stopwatch.GetTimestamp(); ServerApplicationPaths appPaths = StartupHelpers.CreateApplicationPaths(options); - await _setupServer.RunAsync(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost).ConfigureAwait(false); // $JELLYFIN_LOG_DIR needs to be set for the logger configuration manager Environment.SetEnvironmentVariable("JELLYFIN_LOG_DIR", appPaths.LogDirectoryPath); @@ -88,7 +89,8 @@ namespace Jellyfin.Server // Create an instance of the application configuration to use for application startup IConfiguration startupConfig = CreateAppConfiguration(options, appPaths); - + _setupServer = new SetupServer(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost, _loggerFactory, startupConfig); + await _setupServer.RunAsync().ConfigureAwait(false); StartupHelpers.InitializeLoggingFramework(startupConfig, appPaths); _logger = _loggerFactory.CreateLogger("Main"); @@ -120,6 +122,8 @@ namespace Jellyfin.Server } } + StorageHelper.TestCommonPathsForStorageCapacity(appPaths, _loggerFactory.CreateLogger<Startup>()); + StartupHelpers.PerformStaticInitialization(); await Migrations.MigrationRunner.RunPreStartup(appPaths, _loggerFactory).ConfigureAwait(false); @@ -130,10 +134,12 @@ namespace Jellyfin.Server if (_restartOnShutdown) { _startTimestamp = Stopwatch.GetTimestamp(); - _setupServer = new SetupServer(); - await _setupServer.RunAsync(static () => _jellyfinHost?.Services?.GetService<INetworkManager>(), appPaths, static () => _appHost).ConfigureAwait(false); + await _setupServer.StopAsync().ConfigureAwait(false); + await _setupServer.RunAsync().ConfigureAwait(false); } } while (_restartOnShutdown); + + _setupServer.Dispose(); } private static async Task StartServer(IServerApplicationPaths appPaths, StartupOptions options, IConfiguration startupConfig) @@ -170,9 +176,7 @@ namespace Jellyfin.Server try { - await _setupServer.StopAsync().ConfigureAwait(false); - _setupServer.Dispose(); - _setupServer = null!; + await _setupServer!.StopAsync().ConfigureAwait(false); await _jellyfinHost.StartAsync().ConfigureAwait(false); if (!OperatingSystem.IsWindows() && startupConfig.UseUnixSocket()) diff --git a/Jellyfin.Server/ServerSetupApp/SetupServer.cs b/Jellyfin.Server/ServerSetupApp/SetupServer.cs index 9e2cf5bc8..3d4810bd7 100644 --- a/Jellyfin.Server/ServerSetupApp/SetupServer.cs +++ b/Jellyfin.Server/ServerSetupApp/SetupServer.cs @@ -4,6 +4,9 @@ using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; +using Emby.Server.Implementations.Configuration; +using Emby.Server.Implementations.Serialization; +using Jellyfin.Networking.Manager; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Net; using MediaBrowser.Controller; @@ -11,9 +14,12 @@ using MediaBrowser.Model.System; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; namespace Jellyfin.Server.ServerSetupApp; @@ -22,20 +28,45 @@ namespace Jellyfin.Server.ServerSetupApp; /// </summary> public sealed class SetupServer : IDisposable { + private readonly Func<INetworkManager?> _networkManagerFactory; + private readonly IApplicationPaths _applicationPaths; + private readonly Func<IServerApplicationHost?> _serverFactory; + private readonly ILoggerFactory _loggerFactory; + private readonly IConfiguration _startupConfiguration; + private readonly ServerConfigurationManager _configurationManager; private IHost? _startupServer; private bool _disposed; /// <summary> - /// Starts the Bind-All Setup aspcore server to provide a reflection on the current core setup. + /// Initializes a new instance of the <see cref="SetupServer"/> class. /// </summary> /// <param name="networkManagerFactory">The networkmanager.</param> /// <param name="applicationPaths">The application paths.</param> - /// <param name="serverApplicationHost">The servers application host.</param> - /// <returns>A Task.</returns> - public async Task RunAsync( + /// <param name="serverApplicationHostFactory">The servers application host.</param> + /// <param name="loggerFactory">The logger factory.</param> + /// <param name="startupConfiguration">The startup configuration.</param> + public SetupServer( Func<INetworkManager?> networkManagerFactory, IApplicationPaths applicationPaths, - Func<IServerApplicationHost?> serverApplicationHost) + Func<IServerApplicationHost?> serverApplicationHostFactory, + ILoggerFactory loggerFactory, + IConfiguration startupConfiguration) + { + _networkManagerFactory = networkManagerFactory; + _applicationPaths = applicationPaths; + _serverFactory = serverApplicationHostFactory; + _loggerFactory = loggerFactory; + _startupConfiguration = startupConfiguration; + var xmlSerializer = new MyXmlSerializer(); + _configurationManager = new ServerConfigurationManager(_applicationPaths, loggerFactory, xmlSerializer); + _configurationManager.RegisterConfiguration<NetworkConfigurationFactory>(); + } + + /// <summary> + /// Starts the Bind-All Setup aspcore server to provide a reflection on the current core setup. + /// </summary> + /// <returns>A Task.</returns> + public async Task RunAsync() { ThrowIfDisposed(); _startupServer = Host.CreateDefaultBuilder() @@ -48,7 +79,23 @@ public sealed class SetupServer : IDisposable .ConfigureWebHostDefaults(webHostBuilder => { webHostBuilder - .UseKestrel() + .UseKestrel((builderContext, options) => + { + var config = _configurationManager.GetNetworkConfiguration()!; + var knownBindInterfaces = NetworkManager.GetInterfacesCore(_loggerFactory.CreateLogger<SetupServer>(), config.EnableIPv4, config.EnableIPv6); + knownBindInterfaces = NetworkManager.FilterBindSettings(config, knownBindInterfaces.ToList(), config.EnableIPv4, config.EnableIPv6); + var bindInterfaces = NetworkManager.GetAllBindInterfaces(false, _configurationManager, knownBindInterfaces, config.EnableIPv4, config.EnableIPv6); + Extensions.WebHostBuilderExtensions.SetupJellyfinWebServer( + bindInterfaces, + config.InternalHttpPort, + null, + null, + _startupConfiguration, + _applicationPaths, + _loggerFactory.CreateLogger<SetupServer>(), + builderContext, + options); + }) .Configure(app => { app.UseHealthChecks("/health"); @@ -57,14 +104,14 @@ public sealed class SetupServer : IDisposable { loggerRoute.Run(async context => { - var networkManager = networkManagerFactory(); + var networkManager = _networkManagerFactory(); if (context.Connection.RemoteIpAddress is null || networkManager is null || !networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress)) { context.Response.StatusCode = (int)HttpStatusCode.Unauthorized; return; } - var logFilePath = new DirectoryInfo(applicationPaths.LogDirectoryPath) + var logFilePath = new DirectoryInfo(_applicationPaths.LogDirectoryPath) .EnumerateFiles() .OrderBy(f => f.CreationTimeUtc) .FirstOrDefault() @@ -80,20 +127,20 @@ public sealed class SetupServer : IDisposable { systemRoute.Run(async context => { - var jfApplicationHost = serverApplicationHost(); + var jfApplicationHost = _serverFactory(); var retryCounter = 0; while (jfApplicationHost is null && retryCounter < 5) { await Task.Delay(500).ConfigureAwait(false); - jfApplicationHost = serverApplicationHost(); + jfApplicationHost = _serverFactory(); retryCounter++; } if (jfApplicationHost is null) { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new Microsoft.Extensions.Primitives.StringValues("60"); + context.Response.Headers.RetryAfter = new StringValues("5"); return; } @@ -114,9 +161,10 @@ public sealed class SetupServer : IDisposable app.Run((context) => { context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; - context.Response.Headers.RetryAfter = new Microsoft.Extensions.Primitives.StringValues("60"); + context.Response.Headers.RetryAfter = new StringValues("5"); + context.Response.Headers.ContentType = new StringValues("text/html"); context.Response.WriteAsync("<p>Jellyfin Server still starting. Please wait.</p>"); - var networkManager = networkManagerFactory(); + var networkManager = _networkManagerFactory(); if (networkManager is not null && context.Connection.RemoteIpAddress is not null && networkManager.IsInLocalNetwork(context.Connection.RemoteIpAddress)) { context.Response.WriteAsync("<p>You can download the current logfiles <a href='/startup/logger'>here</a>.</p>"); |