diff options
Diffstat (limited to 'Jellyfin.Server/Migrations/Routines')
25 files changed, 1852 insertions, 761 deletions
diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs index 2047ec743..00d152b4b 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to add the default cast receivers to the system config. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T16:00:00", nameof(AddDefaultCastReceivers), "34A1A1C4-5572-418E-A2F8-32CDFE2668E8", RunMigrationOnSetup = true)] public class AddDefaultCastReceivers : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -21,15 +24,6 @@ public class AddDefaultCastReceivers : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("34A1A1C4-5572-418E-A2F8-32CDFE2668E8"); - - /// <inheritdoc /> - public string Name => "AddDefaultCastReceivers"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { _serverConfigurationManager.Configuration.CastReceiverApplications = diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs index fc6b5d597..8c8398a16 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Migration to initialize system configuration with the default plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T09:00:00", nameof(AddDefaultPluginRepository), "EB58EBEE-9514-4B9B-8225-12E1A40020DF", RunMigrationOnSetup = true)] public class AddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -27,15 +30,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("EB58EBEE-9514-4B9B-8225-12E1A40020DF"); - - /// <inheritdoc/> - public string Name => "AddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; - - /// <inheritdoc/> public void Perform() { _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; diff --git a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs index 5a8ef2e1c..1326a6dc8 100644 --- a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs +++ b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs @@ -12,7 +12,10 @@ namespace Jellyfin.Server.Migrations.Routines /// If the deprecated logging.json file exists and has a custom config, it will be used as logging.user.json, /// otherwise a blank file will be created. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T06:00:00", nameof(CreateUserLoggingConfigFile), "EF103419-8451-40D8-9F34-D1A8E93A1679")] internal class CreateUserLoggingConfigFile : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { /// <summary> /// File history for logging.json as existed during this migration creation. The contents for each has been minified. @@ -43,15 +46,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{EF103419-8451-40D8-9F34-D1A8E93A1679}"); - - /// <inheritdoc/> - public string Name => "CreateLoggingConfigHierarchy"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logDirectory = _appPaths.ConfigurationDirectoryPath; diff --git a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs index 378e88e25..acf2835fe 100644 --- a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs +++ b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Disable transcode throttling for all installations since it is currently broken for certain video formats. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T05:00:00", nameof(DisableTranscodingThrottling), "4124C2CD-E939-4FFB-9BE9-9B311C413638")] internal class DisableTranscodingThrottling : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<DisableTranscodingThrottling> _logger; private readonly IConfigurationManager _configManager; @@ -19,15 +22,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{4124C2CD-E939-4FFB-9BE9-9B311C413638}"); - - /// <inheritdoc/> - public string Name => "DisableTranscodingThrottling"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { // Set EnableThrottling to false since it wasn't used before and may introduce issues diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index a20253369..05ded06ba 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -16,9 +16,12 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Fixes the data column of audio types to be deserializable. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T18:00:00", nameof(FixAudioData), "CF6FABC2-9FBE-4933-84A5-FFE52EF22A58")] + [JellyfinMigrationBackup(LegacyLibraryDb = true)] internal class FixAudioData : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private const string DbFilename = "library.db"; private readonly ILogger<FixAudioData> _logger; private readonly IServerApplicationPaths _applicationPaths; private readonly IItemRepository _itemRepository; @@ -34,40 +37,8 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{CF6FABC2-9FBE-4933-84A5-FFE52EF22A58}"); - - /// <inheritdoc/> - public string Name => "FixAudioData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) - { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); - File.Copy(dbPath, bakPath); - _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } - } - _logger.LogInformation("Backfilling audio lyrics data to database."); var startIndex = 0; var records = _itemRepository.GetCount(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/FixDates.cs b/Jellyfin.Server/Migrations/Routines/FixDates.cs new file mode 100644 index 000000000..f112502b9 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/FixDates.cs @@ -0,0 +1,168 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to fix dates saved in the database to always be UTC. +/// </summary> +[JellyfinMigration("2025-06-20T18:00:00", nameof(FixDates))] +public class FixDates : IAsyncMigrationRoutine +{ + private const int PageSize = 5000; + + private readonly ILogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + + /// <summary> + /// Initializes a new instance of the <see cref="FixDates"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI integration.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + public FixDates( + ILogger<FixDates> logger, + IStartupLogger<FixDates> startupLogger, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _logger = startupLogger.With(logger); + _dbProvider = dbProvider; + } + + /// <inheritdoc /> + public async Task PerformAsync(CancellationToken cancellationToken) + { + if (!TimeZoneInfo.Local.Equals(TimeZoneInfo.Utc)) + { + using var context = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var sw = Stopwatch.StartNew(); + + await FixBaseItemsAsync(context, sw, cancellationToken).ConfigureAwait(false); + sw.Reset(); + await FixChaptersAsync(context, sw, cancellationToken).ConfigureAwait(false); + sw.Reset(); + await FixBaseItemImageInfos(context, sw, cancellationToken).ConfigureAwait(false); + } + } + + private async Task FixBaseItemsAsync(JellyfinDbContext context, Stopwatch sw, CancellationToken cancellationToken) + { + int itemCount = 0; + + var baseQuery = context.BaseItems.OrderBy(e => e.Id); + var records = baseQuery.Count(); + _logger.LogInformation("Fixing dates for {Count} BaseItems.", records); + + sw.Start(); + await foreach (var result in context.BaseItems.OrderBy(e => e.Id) + .WithPartitionProgress( + (partition) => + _logger.LogInformation( + "Processing BaseItems batch {BatchNumber} ({ProcessedSoFar}/{TotalRecords}) - Time: {ElapsedTime}", + partition + 1, + Math.Min((partition + 1) * PageSize, records), + records, + sw.Elapsed)) + .PartitionEagerAsync(PageSize, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + result.DateCreated = ToUniversalTime(result.DateCreated); + result.DateLastMediaAdded = ToUniversalTime(result.DateLastMediaAdded); + result.DateLastRefreshed = ToUniversalTime(result.DateLastRefreshed); + result.DateLastSaved = ToUniversalTime(result.DateLastSaved); + result.DateModified = ToUniversalTime(result.DateModified); + itemCount++; + } + + var saveCount = await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation("BaseItems: Processed {ItemCount} items, saved {SaveCount} changes in {ElapsedTime}", itemCount, saveCount, sw.Elapsed); + } + + private async Task FixChaptersAsync(JellyfinDbContext context, Stopwatch sw, CancellationToken cancellationToken) + { + int itemCount = 0; + + var baseQuery = context.Chapters; + var records = baseQuery.Count(); + _logger.LogInformation("Fixing dates for {Count} Chapters.", records); + + sw.Start(); + await foreach (var result in context.Chapters.OrderBy(e => e.ItemId) + .WithPartitionProgress( + (partition) => + _logger.LogInformation( + "Processing Chapter batch {BatchNumber} ({ProcessedSoFar}/{TotalRecords}) - Time: {ElapsedTime}", + partition + 1, + Math.Min((partition + 1) * PageSize, records), + records, + sw.Elapsed)) + .PartitionEagerAsync(PageSize, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + result.ImageDateModified = ToUniversalTime(result.ImageDateModified, true); + itemCount++; + } + + var saveCount = await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Chapters: Processed {ItemCount} items, saved {SaveCount} changes in {ElapsedTime}", itemCount, saveCount, sw.Elapsed); + } + + private async Task FixBaseItemImageInfos(JellyfinDbContext context, Stopwatch sw, CancellationToken cancellationToken) + { + int itemCount = 0; + + var baseQuery = context.BaseItemImageInfos; + var records = baseQuery.Count(); + _logger.LogInformation("Fixing dates for {Count} BaseItemImageInfos.", records); + + sw.Start(); + await foreach (var result in context.BaseItemImageInfos.OrderBy(e => e.Id) + .WithPartitionProgress( + (partition) => + _logger.LogInformation( + "Processing BaseItemImageInfos batch {BatchNumber} ({ProcessedSoFar}/{TotalRecords}) - Time: {ElapsedTime}", + partition + 1, + Math.Min((partition + 1) * PageSize, records), + records, + sw.Elapsed)) + .PartitionEagerAsync(PageSize, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + result.DateModified = ToUniversalTime(result.DateModified); + itemCount++; + } + + var saveCount = await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation("BaseItemImageInfos: Processed {ItemCount} items, saved {SaveCount} changes in {ElapsedTime}", itemCount, saveCount, sw.Elapsed); + } + + private DateTime? ToUniversalTime(DateTime? dateTime, bool isUTC = false) + { + if (dateTime is null) + { + return null; + } + + if (dateTime.Value.Year == 1 && dateTime.Value.Month == 1 && dateTime.Value.Day == 1) + { + return null; + } + + if (dateTime.Value.Kind == DateTimeKind.Utc || isUTC) + { + return dateTime.Value; + } + + return dateTime.Value.ToUniversalTime(); + } +} diff --git a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs index 192c170b2..56614ece3 100644 --- a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs +++ b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs @@ -13,7 +13,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Properly set playlist owner. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T15:00:00", nameof(FixPlaylistOwner), "615DFA9E-2497-4DBB-A472-61938B752C5B")] internal class FixPlaylistOwner : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<FixPlaylistOwner> _logger; private readonly ILibraryManager _libraryManager; @@ -30,15 +33,6 @@ internal class FixPlaylistOwner : IMigrationRoutine } /// <inheritdoc/> - public Guid Id => Guid.Parse("{615DFA9E-2497-4DBB-A472-61938B752C5B}"); - - /// <inheritdoc/> - public string Name => "FixPlaylistOwner"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs index 2f23cb1f8..a954d307e 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs @@ -2,8 +2,8 @@ using System; using System.Collections.Generic; using System.IO; using Emby.Server.Implementations.Data; -using Jellyfin.Data.Entities; -using Jellyfin.Server.Implementations; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; using MediaBrowser.Controller; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; @@ -14,7 +14,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the activity log database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T07:00:00", nameof(MigrateActivityLogDb), "3793eb59-bc8c-456c-8b9f-bd5a62a42978")] public class MigrateActivityLogDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "activitylog.db"; @@ -36,15 +39,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("3793eb59-bc8c-456c-8b9f-bd5a62a42978"); - - /// <inheritdoc/> - public string Name => "MigrateActivityLogDatabase"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logLevelDictionary = new Dictionary<string, LogLevel>(StringComparer.OrdinalIgnoreCase) diff --git a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs index c845beef2..c6699c21d 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs @@ -2,8 +2,8 @@ using System; using System.Collections.Generic; using System.IO; using Emby.Server.Implementations.Data; -using Jellyfin.Data.Entities.Security; -using Jellyfin.Server.Implementations; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities.Security; using MediaBrowser.Controller; using MediaBrowser.Controller.Library; using Microsoft.Data.Sqlite; @@ -15,7 +15,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// A migration that moves data from the authentication database into the new schema. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T14:00:00", nameof(MigrateAuthenticationDb), "5BD72F41-E6F3-4F60-90AA-09869ABE0E22")] public class MigrateAuthenticationDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "authentication.db"; @@ -44,15 +47,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("5BD72F41-E6F3-4F60-90AA-09869ABE0E22"); - - /// <inheritdoc /> - public string Name => "MigrateAuthenticationDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { var dataPath = _appPaths.DataPath; diff --git a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs index 502a37cde..0d9952ce9 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs @@ -5,9 +5,9 @@ using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using Emby.Server.Implementations.Data; -using Jellyfin.Data.Entities; -using Jellyfin.Data.Enums; -using Jellyfin.Server.Implementations; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Database.Implementations.Enums; using MediaBrowser.Controller; using MediaBrowser.Controller.Library; using MediaBrowser.Model.Dto; @@ -20,7 +20,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the display preferences database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T12:00:00", nameof(MigrateDisplayPreferencesDb), "06387815-C3CC-421F-A888-FB5F9992BEA8")] public class MigrateDisplayPreferencesDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "displaypreferences.db"; @@ -52,15 +55,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("06387815-C3CC-421F-A888-FB5F9992BEA8"); - - /// <inheritdoc /> - public string Name => "MigrateDisplayPreferencesDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { HomeSectionType[] defaults = diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs new file mode 100644 index 000000000..c199ee4d6 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -0,0 +1,152 @@ +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions.Json; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-21T00:00:00", nameof(MigrateKeyframeData))] +public class MigrateKeyframeData : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IApplicationPaths _appPaths; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="dbProvider">The EFCore db factory.</param> + public MigrateKeyframeData( + IStartupLogger<MigrateKeyframeData> startupLogger, + IApplicationPaths appPaths, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _logger = startupLogger; + _appPaths = appPaths; + _dbProvider = dbProvider; + } + + private string KeyframeCachePath => Path.Combine(_appPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var baseQuery = context.BaseItems.Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder).OrderBy(e => e.Id); + var records = baseQuery.Count(); + _logger.LogInformation("Checking {Count} items for importable keyframe data.", records); + + context.KeyframeData.ExecuteDelete(); + using var transaction = context.Database.BeginTransaction(); + do + { + var results = baseQuery.Skip(offset).Take(Limit).Select(b => new Tuple<Guid, string?>(b.Id, b.Path)).ToList(); + foreach (var result in results) + { + if (TryGetKeyframeData(result.Item1, result.Item2, out var data)) + { + itemCount++; + context.KeyframeData.Add(data); + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Imported: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + transaction.Commit(); + + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + + if (Directory.Exists(KeyframeCachePath)) + { + Directory.Delete(KeyframeCachePath, true); + } + } + + private bool TryGetKeyframeData(Guid id, string? path, [NotNullWhen(true)] out KeyframeData? data) + { + data = null; + if (!string.IsNullOrEmpty(path)) + { + var cachePath = GetCachePath(KeyframeCachePath, path); + if (TryReadFromCache(cachePath, out var keyframeData)) + { + data = new() + { + ItemId = id, + KeyframeTicks = keyframeData.KeyframeTicks.ToList(), + TotalDuration = keyframeData.TotalDuration + }; + + return true; + } + } + + return false; + } + + private string? GetCachePath(string keyframeCachePath, string filePath) + { + DateTime? lastWriteTimeUtc; + try + { + lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping {Path}: {Exception}", filePath, e.Message); + + return null; + } + + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string? cachePath, [NotNullWhen(true)] out MediaEncoding.Keyframes.KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<MediaEncoding.Keyframes.KeyframeData>(bytes, _jsonOptions); + + return cachedResult is not null; + } + + cachedResult = null; + + return false; + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index 3289484f9..e04a2737a 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -10,336 +10,415 @@ using System.IO; using System.Linq; using System.Text; using Emby.Server.Implementations.Data; -using Jellyfin.Data.Entities; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; -using Jellyfin.Server.Implementations; using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using Chapter = Jellyfin.Data.Entities.Chapter; +using BaseItemEntity = Jellyfin.Database.Implementations.Entities.BaseItemEntity; +using Chapter = Jellyfin.Database.Implementations.Entities.Chapter; namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// The migration routine for migrating the userdata database to EF Core. /// </summary> -public class MigrateLibraryDb : IMigrationRoutine +[JellyfinMigration("2025-04-20T20:00:00", nameof(MigrateLibraryDb))] +[JellyfinMigrationBackup(JellyfinDb = true, LegacyLibraryDb = true)] +internal class MigrateLibraryDb : IDatabaseMigrationRoutine { private const string DbFilename = "library.db"; - private readonly ILogger<MigrateLibraryDb> _logger; + private readonly IStartupLogger _logger; private readonly IServerApplicationPaths _paths; + private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; private readonly IDbContextFactory<JellyfinDbContext> _provider; /// <summary> /// Initializes a new instance of the <see cref="MigrateLibraryDb"/> class. /// </summary> - /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="provider">The database provider.</param> /// <param name="paths">The server application paths.</param> + /// <param name="jellyfinDatabaseProvider">The database provider for special access.</param> public MigrateLibraryDb( - ILogger<MigrateLibraryDb> logger, + IStartupLogger<MigrateLibraryDb> startupLogger, IDbContextFactory<JellyfinDbContext> provider, - IServerApplicationPaths paths) + IServerApplicationPaths paths, + IJellyfinDatabaseProvider jellyfinDatabaseProvider) { - _logger = logger; + _logger = startupLogger; _provider = provider; _paths = paths; + _jellyfinDatabaseProvider = jellyfinDatabaseProvider; } /// <inheritdoc/> - public Guid Id => Guid.Parse("36445464-849f-429f-9ad0-bb130efa0664"); - - /// <inheritdoc/> - public string Name => "MigrateLibraryDbData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; // TODO Change back after testing - - /// <inheritdoc/> public void Perform() { _logger.LogInformation("Migrating the userdata from library.db may take a while, do not stop Jellyfin."); var dataPath = _paths.DataPath; var libraryDbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={libraryDbPath}"); - var migrationTotalTime = TimeSpan.Zero; + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } - var stopwatch = new Stopwatch(); - stopwatch.Start(); + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving TypedBaseItem."); - const string typedBaseItemsQuery = """ - SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, - IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, - PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, - ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, - Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, - DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, - PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, - ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType FROM TypedBaseItems - """; - dbContext.BaseItems.ExecuteDelete(); + var fullOperationTimer = new Stopwatch(); + fullOperationTimer.Start(); - var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); - foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + using (var operation = GetPreparedDbContext("Cleanup database")) { - var baseItem = GetItem(dto); - dbContext.BaseItems.Add(baseItem.BaseItem); - foreach (var dataKey in baseItem.LegacyUserDataKey) - { - legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; - } + operation.JellyfinDbContext.AttachmentStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.BaseItems.ExecuteDelete(); + operation.JellyfinDbContext.ItemValues.ExecuteDelete(); + operation.JellyfinDbContext.UserData.ExecuteDelete(); + operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.Peoples.ExecuteDelete(); + operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete(); + operation.JellyfinDbContext.Chapters.ExecuteDelete(); + operation.JellyfinDbContext.AncestorIds.ExecuteDelete(); } - _logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + // notify the other migration to just silently abort because the fix has been applied here already. + ReseedFolderFlag.RerunGuardFlag = true; - _logger.LogInformation("Start moving ItemValues."); - // do not migrate inherited types as they are now properly mapped in search and lookup. - const string itemValueQuery = - """ - SELECT ItemId, Type, Value, CleanValue FROM ItemValues - WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) - """; - dbContext.ItemValues.ExecuteDelete(); - - // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. - var localItems = new Dictionary<(int Type, string CleanValue), (ItemValue ItemValue, List<Guid> ItemIds)>(); + var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); + connection.Open(); - foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) - { - var itemId = dto.GetGuid(0); - var entity = GetItemValue(dto); - var key = ((int)entity.Type, entity.CleanValue); - if (!localItems.TryGetValue(key, out var existing)) + var baseItemIds = new HashSet<Guid>(); + using (var operation = GetPreparedDbContext("Moving TypedBaseItem")) + { + const string typedBaseItemsQuery = + """ + SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, + IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, + PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, + ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, + Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, + DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, + PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, + ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType, IsFolder FROM TypedBaseItems + """; + using (new TrackedMigrationStep("Loading TypedBaseItems", _logger)) { - localItems[key] = existing = (entity, []); + foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + { + var baseItem = GetItem(dto); + operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem); + baseItemIds.Add(baseItem.BaseItem.Id); + foreach (var dataKey in baseItem.LegacyUserDataKey) + { + legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + } + } } - existing.ItemIds.Add(itemId); - } - - foreach (var item in localItems) - { - dbContext.ItemValues.Add(item.Value.ItemValue); - dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger)) { - Item = null!, - ItemValue = null!, - ItemId = f, - ItemValueId = item.Value.ItemValue.ItemValueId - })); + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving UserData."); - var queryResult = connection.Query(""" - SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas - - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) - """); + using (var operation = GetPreparedDbContext("Moving ItemValues")) + { + // do not migrate inherited types as they are now properly mapped in search and lookup. + const string itemValueQuery = + """ + SELECT ItemId, Type, Value, CleanValue FROM ItemValues + WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) + """; - dbContext.UserData.ExecuteDelete(); + // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. + var localItems = new Dictionary<(int Type, string Value), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep("Loading ItemValues", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + { + var itemId = dto.GetGuid(0); + var entity = GetItemValue(dto); + var key = ((int)entity.Type, entity.Value); + if (!localItems.TryGetValue(key, out var existing)) + { + localItems[key] = existing = (entity, []); + } - var users = dbContext.Users.AsNoTracking().ToImmutableArray(); - var oldUserdata = new Dictionary<string, UserData>(); + existing.ItemIds.Add(itemId); + } - foreach (var entity in queryResult) - { - var userData = GetUserData(users, entity); - if (userData is null) - { - _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0)); - continue; + foreach (var item in localItems) + { + operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue); + operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + { + Item = null!, + ItemValue = null!, + ItemId = f, + ItemValueId = item.Value.ItemValue.ItemValueId + })); + } } - if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger)) { - _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); - continue; + operation.JellyfinDbContext.SaveChanges(); } - - userData.ItemId = refItem.Id; - dbContext.UserData.Add(userData); } - _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count); - dbContext.SaveChanges(); + using (var operation = GetPreparedDbContext("Moving UserData")) + { + var queryResult = connection.Query( + """ + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas - _logger.LogInformation("Start moving MediaStreamInfos."); - const string mediaStreamQuery = """ - SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, - IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, - AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, - Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, - DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired - FROM MediaStreams - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) - """; - dbContext.MediaStreamInfos.ExecuteDelete(); + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) + """); - foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) - { - dbContext.MediaStreamInfos.Add(GetMediaStream(dto)); - } + using (new TrackedMigrationStep("Loading UserData", _logger)) + { + var users = operation.JellyfinDbContext.Users.AsNoTracking().ToArray(); + var userIdBlacklist = new HashSet<int>(); - _logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); + foreach (var entity in queryResult) + { + var userData = GetUserData(users, entity, userIdBlacklist, _logger); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } - _logger.LogInformation("Start moving People."); - const string personsQuery = """ - SELECT ItemId, Name, Role, PersonType, SortOrder FROM People - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) - """; - dbContext.Peoples.ExecuteDelete(); - dbContext.PeopleBaseItemMap.ExecuteDelete(); + continue; + } - var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); + if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + { + _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); + continue; + } - foreach (SqliteDataReader reader in connection.Query(personsQuery)) - { - var itemId = reader.GetGuid(0); - if (!dbContext.BaseItems.Any(f => f.Id == itemId)) + userData.ItemId = refItem.Id; + operation.JellyfinDbContext.UserData.Add(userData); + } + } + + legacyBaseItemWithUserKeys.Clear(); + + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger)) { - _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); - continue; + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("Moving MediaStreamInfos")) + { + const string mediaStreamQuery = + """ + SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, + IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, + AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, + Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, + DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired + FROM MediaStreams + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) + """; - var entity = GetPerson(reader); - if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + using (new TrackedMigrationStep("Loading MediaStreamInfos", _logger)) { - peopleCache[entity.Name] = personCache = (entity, []); + foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + { + operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto)); + } } - if (reader.TryGetString(2, out var role)) + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger)) { + operation.JellyfinDbContext.SaveChanges(); } + } - if (reader.TryGetInt32(4, out var sortOrder)) + using (var operation = GetPreparedDbContext("Moving AttachmentStreamInfos")) + { + const string mediaAttachmentQuery = + """ + SELECT ItemId, AttachmentIndex, Codec, CodecTag, Comment, filename, MIMEType + FROM mediaattachments + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = mediaattachments.ItemId) + """; + + using (new TrackedMigrationStep("Loading AttachmentStreamInfos", _logger)) { + foreach (SqliteDataReader dto in connection.Query(mediaAttachmentQuery)) + { + operation.JellyfinDbContext.AttachmentStreamInfos.Add(GetMediaAttachment(dto)); + } } - personCache.Items.Add(new PeopleBaseItemMap() + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.AttachmentStreamInfos.Local.Count} AttachmentStreamInfos entries", _logger)) { - Item = null!, - ItemId = itemId, - People = null!, - PeopleId = personCache.Person.Id, - ListOrder = sortOrder, - SortOrder = sortOrder, - Role = role - }); + operation.JellyfinDbContext.SaveChanges(); + } } - foreach (var item in peopleCache) + using (var operation = GetPreparedDbContext("Moving People")) { - dbContext.Peoples.Add(item.Value.Person); - dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); - } + const string personsQuery = + """ + SELECT ItemId, Name, Role, PersonType, SortOrder FROM People + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) + """; - _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - _logger.LogInformation("Start moving Chapters."); - const string chapterQuery = """ - SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) - """; - dbContext.Chapters.ExecuteDelete(); + using (new TrackedMigrationStep("Loading People", _logger)) + { + foreach (SqliteDataReader reader in connection.Query(personsQuery)) + { + var itemId = reader.GetGuid(0); + if (!baseItemIds.Contains(itemId)) + { + _logger.LogError("Not saving person {0} because it's not in use by any BaseItem", reader.GetString(1)); + continue; + } - foreach (SqliteDataReader dto in connection.Query(chapterQuery)) - { - var chapter = GetChapter(dto); - dbContext.Chapters.Add(chapter); - } + var entity = GetPerson(reader); + if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + { + peopleCache[entity.Name] = personCache = (entity, []); + } + + if (reader.TryGetString(2, out var role)) + { + } + + int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); + + personCache.Items.Add(new PeopleBaseItemMap() + { + Item = null!, + ItemId = itemId, + People = null!, + PeopleId = personCache.Person.Id, + ListOrder = sortOrder, + SortOrder = sortOrder, + Role = role + }); + } + + baseItemIds.Clear(); - _logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + foreach (var item in peopleCache) + { + operation.JellyfinDbContext.Peoples.Add(item.Value.Person); + operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); + } - _logger.LogInformation("Start moving AncestorIds."); - const string ancestorIdsQuery = """ - SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds - WHERE - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) - AND - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) - """; - dbContext.Chapters.ExecuteDelete(); + peopleCache.Clear(); + } - foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } + + using (var operation = GetPreparedDbContext("Moving Chapters")) { - var ancestorId = GetAncestorId(dto); - dbContext.AncestorIds.Add(ancestorId); + const string chapterQuery = + """ + SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) + """; + + using (new TrackedMigrationStep("Loading Chapters", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(chapterQuery)) + { + var chapter = GetChapter(dto); + operation.JellyfinDbContext.Chapters.Add(chapter); + } + } + + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.Chapters.Local.Count); + using (var operation = GetPreparedDbContext("Moving AncestorIds")) + { + const string ancestorIdsQuery = + """ + SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds + WHERE + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) + AND + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) + """; + + using (new TrackedMigrationStep("Loading AncestorIds", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + { + var ancestorId = GetAncestorId(dto); + operation.JellyfinDbContext.AncestorIds.Add(ancestorId); + } + } - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (new TrackedMigrationStep($"Saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } connection.Close(); + _logger.LogInformation("Migration of the Library.db done."); - _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + _logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed); SqliteConnection.ClearAllPools(); - File.Move(libraryDbPath, libraryDbPath + ".old"); - _logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime); + _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + File.Move(libraryDbPath, libraryDbPath + ".old", true); + } - if (dbContext.Database.IsSqlite()) - { - _logger.LogInformation("Vacuum and Optimise jellyfin.db now."); - dbContext.Database.ExecuteSqlRaw("PRAGMA optimize"); - dbContext.Database.ExecuteSqlRaw("VACUUM"); - _logger.LogInformation("jellyfin.db optimized successfully!"); - } - else - { - _logger.LogInformation("This database doesn't support optimization"); - } + private DatabaseMigrationStep GetPreparedDbContext(string operationName) + { + var dbContext = _provider.CreateDbContext(); + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return new DatabaseMigrationStep(dbContext, operationName, _logger); } - private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto) + internal static UserData? GetUserData(User[] users, SqliteDataReader dto, HashSet<int> userIdBlacklist, ILogger logger) { var internalUserId = dto.GetInt32(1); - var user = users.FirstOrDefault(e => e.InternalId == internalUserId); + if (userIdBlacklist.Contains(internalUserId)) + { + return null; + } + var user = users.FirstOrDefault(e => e.InternalId == internalUserId); if (user is null) { - _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); + userIdBlacklist.Add(internalUserId); + + logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); return null; } @@ -653,6 +732,48 @@ public class MigrateLibraryDb : IMigrationRoutine return item; } + /// <summary> + /// Gets the attachment. + /// </summary> + /// <param name="reader">The reader.</param> + /// <returns>MediaAttachment.</returns> + private AttachmentStreamInfo GetMediaAttachment(SqliteDataReader reader) + { + var item = new AttachmentStreamInfo + { + Index = reader.GetInt32(1), + Item = null!, + ItemId = reader.GetGuid(0), + }; + + if (reader.TryGetString(2, out var codec)) + { + item.Codec = codec; + } + + if (reader.TryGetString(3, out var codecTag)) + { + item.CodecTag = codecTag; + } + + if (reader.TryGetString(4, out var comment)) + { + item.Comment = comment; + } + + if (reader.TryGetString(5, out var fileName)) + { + item.Filename = fileName; + } + + if (reader.TryGetString(6, out var mimeType)) + { + item.MimeType = mimeType; + } + + return item; + } + private (BaseItemEntity BaseItem, string[] LegacyUserDataKey) GetItem(SqliteDataReader reader) { var entity = new BaseItemEntity() @@ -1034,7 +1155,27 @@ public class MigrateLibraryDb : IMigrationRoutine entity.MediaType = mediaType; } - var baseItem = BaseItemRepository.DeserialiseBaseItem(entity, _logger, null, false); + if (reader.TryGetString(index++, out var sortName)) + { + entity.SortName = sortName; + } + + if (reader.TryGetString(index++, out var cleanName)) + { + entity.CleanName = cleanName; + } + + if (reader.TryGetString(index++, out var unratedType)) + { + entity.UnratedType = unratedType; + } + + if (reader.TryGetBoolean(index++, out var isFolder)) + { + entity.IsFolder = isFolder; + } + + var baseItem = BaseItemRepository.DeserializeBaseItem(entity, _logger, null, false); var dataKeys = baseItem.GetUserDataKeys(); userDataKeys.AddRange(dataKeys); @@ -1198,4 +1339,58 @@ public class MigrateLibraryDb : IMigrationRoutine return image; } + + private class TrackedMigrationStep : IDisposable + { + private readonly string _operationName; + private readonly ILogger _logger; + private readonly Stopwatch _operationTimer; + private bool _disposed; + + public TrackedMigrationStep(string operationName, ILogger logger) + { + _operationName = operationName; + _logger = logger; + _operationTimer = Stopwatch.StartNew(); + logger.LogInformation("Start {OperationName}", operationName); + } + + public bool Disposed + { + get => _disposed; + set => _disposed = value; + } + + public virtual void Dispose() + { + if (Disposed) + { + return; + } + + Disposed = true; + _logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed); + } + } + + private sealed class DatabaseMigrationStep : TrackedMigrationStep + { + public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger) + { + JellyfinDbContext = jellyfinDbContext; + } + + public JellyfinDbContext JellyfinDbContext { get; } + + public override void Dispose() + { + if (Disposed) + { + return; + } + + JellyfinDbContext.Dispose(); + base.Dispose(); + } + } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs new file mode 100644 index 000000000..d4cc9bbee --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs @@ -0,0 +1,73 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for checking if the current instance of Jellyfin is compatiable to be upgraded. +/// </summary> +[JellyfinMigration("2025-04-20T19:30:00", nameof(MigrateLibraryDbCompatibilityCheck))] +public class MigrateLibraryDbCompatibilityCheck : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db"; + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateLibraryDbCompatibilityCheck"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger.</param> + /// <param name="paths">The Path service.</param> + public MigrateLibraryDbCompatibilityCheck(IStartupLogger<MigrateLibraryDbCompatibilityCheck> startupLogger, IServerApplicationPaths paths) + { + _logger = startupLogger; + _paths = paths; + } + + /// <inheritdoc/> + public async Task PerformAsync(CancellationToken cancellationToken) + { + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } + + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + CheckMigratableVersion(connection); + await connection.CloseAsync().ConfigureAwait(false); + } + + private static void CheckMigratableVersion(SqliteConnection connection) + { + CheckColumnExistance(connection, "TypedBaseItems", "lufs"); + CheckColumnExistance(connection, "TypedBaseItems", "normalizationgain"); + CheckColumnExistance(connection, "mediastreams", "dvversionmajor"); + + static void CheckColumnExistance(SqliteConnection connection, string table, string column) + { + using (var cmd = connection.CreateCommand()) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + cmd.CommandText = $"Select COUNT(1) FROM pragma_table_xinfo('{table}') WHERE lower(name) = '{column}';"; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + var result = cmd.ExecuteScalar()!; + if (!result.Equals(1L)) + { + throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or above are supported. Please upgrade first to server version 10.10.7 before attempting to upgrade afterwards to 10.11"); + } + } + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryUserData.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryUserData.cs new file mode 100644 index 000000000..8a0a1741f --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryUserData.cs @@ -0,0 +1,123 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.Data; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +[JellyfinMigration("2025-06-18T01:00:00", nameof(MigrateLibraryUserData))] +[JellyfinMigrationBackup(JellyfinDb = true)] +internal class MigrateLibraryUserData : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db.old"; + + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + + public MigrateLibraryUserData( + IStartupLogger<MigrateLibraryDb> startupLogger, + IDbContextFactory<JellyfinDbContext> provider, + IServerApplicationPaths paths) + { + _logger = startupLogger; + _provider = provider; + _paths = paths; + } + + public async Task PerformAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Migrating the userdata from library.db.old may take a while, do not stop Jellyfin."); + + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate userdata from {LibraryDb} as it does not exist. This migration expects the MigrateLibraryDb to run first.", libraryDbPath); + return; + } + + var dbContext = await _provider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + if (!await dbContext.BaseItems.AnyAsync(e => e.Id == BaseItemRepository.PlaceholderId, cancellationToken).ConfigureAwait(false)) + { + // the placeholder baseitem has been deleted by the librarydb migration so we need to readd it. + await dbContext.BaseItems.AddAsync( + new Database.Implementations.Entities.BaseItemEntity() + { + Id = BaseItemRepository.PlaceholderId, + Type = "PLACEHOLDER", + Name = "This is a placeholder item for UserData that has been detacted from its original item" + }, + cancellationToken) + .ConfigureAwait(false); + await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + + var users = dbContext.Users.AsNoTracking().ToArray(); + var userIdBlacklist = new HashSet<int>(); + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + var retentionDate = DateTime.UtcNow; + + var queryResult = connection.Query( +""" + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + + WHERE NOT EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) +"""); + + var importedUserData = new Dictionary<Guid, List<UserData>>(); + foreach (var entity in queryResult) + { + var userData = MigrateLibraryDb.GetUserData(users, entity, userIdBlacklist, _logger); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); + + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } + + continue; + } + + var ogId = userData.ItemId; + userData.ItemId = BaseItemRepository.PlaceholderId; + userData.RetentionDate = retentionDate; + if (!importedUserData.TryGetValue(ogId, out var importUserData)) + { + importUserData = []; + importedUserData[ogId] = importUserData; + } + + importUserData.Add(userData); + } + + foreach (var item in importedUserData) + { + await dbContext.UserData.Where(e => e.ItemId == item.Key).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + dbContext.UserData.AddRange(item.Value.DistinctBy(e => e.CustomDataKey)); // old userdata can have fucked up duplicates + } + + _logger.LogInformation("Try saving {NewSaved} UserData entries.", dbContext.UserData.Local.Count); + await dbContext.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 9c2184029..2a6db01cf 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,94 +1,69 @@ using System; -using System.Globalization; -using System.IO; -using Emby.Server.Implementations.Data; -using MediaBrowser.Controller; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Model.Globalization; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Migrate rating levels to new rating level system. - /// </summary> - internal class MigrateRatingLevels : IMigrationRoutine - { - private const string DbFilename = "library.db"; - private readonly ILogger<MigrateRatingLevels> _logger; - private readonly IServerApplicationPaths _applicationPaths; - private readonly ILocalizationManager _localizationManager; - - public MigrateRatingLevels( - IServerApplicationPaths applicationPaths, - ILoggerFactory loggerFactory, - ILocalizationManager localizationManager) - { - _applicationPaths = applicationPaths; - _localizationManager = localizationManager; - _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{73DAB92A-178B-48CD-B05B-FE18733ACDC8}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "MigrateRatingLevels"; +/// <summary> +/// Migrate rating levels. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T22:00:00", nameof(MigrateRatingLevels))] +[JellyfinMigrationBackup(JellyfinDb = true)] +#pragma warning restore CS0618 // Type or member is obsolete +internal class MigrateRatingLevels : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly ILocalizationManager _localizationManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public MigrateRatingLevels( + IDbContextFactory<JellyfinDbContext> provider, + IStartupLogger<MigrateRatingLevels> logger, + ILocalizationManager localizationManager) + { + _provider = provider; + _localizationManager = localizationManager; + _logger = logger; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + _logger.LogInformation("Recalculating parental rating levels based on rating string."); + using var context = _provider.CreateDbContext(); + using var transaction = context.Database.BeginTransaction(); + var ratings = context.BaseItems.AsNoTracking().Select(e => e.OfficialRating).Distinct(); + foreach (var rating in ratings) { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) + if (string.IsNullOrEmpty(rating)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } + int? value = null; + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, value)); + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, value)); } - - // Migrate parental rating strings to new levels - _logger.LogInformation("Recalculating parental rating levels based on rating string."); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + else { - var queryResult = connection.Query("SELECT DISTINCT OfficialRating FROM TypedBaseItems"); - foreach (var entry in queryResult) - { - if (!entry.TryGetString(0, out var ratingString) || string.IsNullOrEmpty(ratingString)) - { - connection.Execute("UPDATE TypedBaseItems SET InheritedParentalRatingValue = NULL WHERE OfficialRating IS NULL OR OfficialRating='';"); - } - else - { - var ratingValue = _localizationManager.GetRatingLevel(ratingString)?.ToString(CultureInfo.InvariantCulture) ?? "NULL"; - - using var statement = connection.PrepareStatement("UPDATE TypedBaseItems SET InheritedParentalRatingValue = @Value WHERE OfficialRating = @Rating;"); - statement.TryBind("@Value", ratingValue); - statement.TryBind("@Rating", ratingString); - statement.ExecuteNonQuery(); - } - } - - transaction.Commit(); + var ratingValue = _localizationManager.GetRatingScore(rating); + var score = ratingValue?.Score; + var subScore = ratingValue?.SubScore; + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, score)); + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, subScore)); } } + + transaction.Commit(); } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs index 7dcae5bd9..e5584fb94 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs @@ -1,10 +1,11 @@ using System; using System.IO; using Emby.Server.Implementations.Data; -using Jellyfin.Data.Entities; -using Jellyfin.Data.Enums; +using Jellyfin.Data; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Database.Implementations.Enums; using Jellyfin.Extensions.Json; -using Jellyfin.Server.Implementations; using Jellyfin.Server.Implementations.Users; using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; @@ -16,206 +17,200 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using JsonSerializer = System.Text.Json.JsonSerializer; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for migrating the user database to EF Core. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T10:00:00", nameof(MigrateUserDb), "5C4B82A2-F053-4009-BD05-B6FCAD82F14C")] +public class MigrateUserDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private const string DbFilename = "users.db"; + + private readonly ILogger<MigrateUserDb> _logger; + private readonly IServerApplicationPaths _paths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly IXmlSerializer _xmlSerializer; + /// <summary> - /// The migration routine for migrating the user database to EF Core. + /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. /// </summary> - public class MigrateUserDb : IMigrationRoutine + /// <param name="logger">The logger.</param> + /// <param name="paths">The server application paths.</param> + /// <param name="provider">The database provider.</param> + /// <param name="xmlSerializer">The xml serializer.</param> + public MigrateUserDb( + ILogger<MigrateUserDb> logger, + IServerApplicationPaths paths, + IDbContextFactory<JellyfinDbContext> provider, + IXmlSerializer xmlSerializer) { - private const string DbFilename = "users.db"; - - private readonly ILogger<MigrateUserDb> _logger; - private readonly IServerApplicationPaths _paths; - private readonly IDbContextFactory<JellyfinDbContext> _provider; - private readonly IXmlSerializer _xmlSerializer; - - /// <summary> - /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. - /// </summary> - /// <param name="logger">The logger.</param> - /// <param name="paths">The server application paths.</param> - /// <param name="provider">The database provider.</param> - /// <param name="xmlSerializer">The xml serializer.</param> - public MigrateUserDb( - ILogger<MigrateUserDb> logger, - IServerApplicationPaths paths, - IDbContextFactory<JellyfinDbContext> provider, - IXmlSerializer xmlSerializer) - { - _logger = logger; - _paths = paths; - _provider = provider; - _xmlSerializer = xmlSerializer; - } + _logger = logger; + _paths = paths; + _provider = provider; + _xmlSerializer = xmlSerializer; + } - /// <inheritdoc/> - public Guid Id => Guid.Parse("5C4B82A2-F053-4009-BD05-B6FCAD82F14C"); + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); - /// <inheritdoc/> - public string Name => "MigrateUserDatabase"; + using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + { + connection.Open(); + using var dbContext = _provider.CreateDbContext(); - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); + dbContext.RemoveRange(dbContext.Users); + dbContext.SaveChanges(); - using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + foreach (var entry in queryResult) { - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); + UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); + if (mockup is null) + { + continue; + } - dbContext.RemoveRange(dbContext.Users); - dbContext.SaveChanges(); + var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); + + var configPath = Path.Combine(userDataDir, "config.xml"); + var config = File.Exists(configPath) + ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() + : new UserConfiguration(); + + var policyPath = Path.Combine(userDataDir, "policy.xml"); + var policy = File.Exists(policyPath) + ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() + : new UserPolicy(); + policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( + "Emby.Server.Implementations.Library", + "Jellyfin.Server.Implementations.Users", + StringComparison.Ordinal) + ?? typeof(DefaultAuthenticationProvider).FullName; + + policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; + int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch + { + -1 => null, + 0 => 3, + _ => policy.LoginAttemptsBeforeLockout + }; - foreach (var entry in queryResult) + var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) { - UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); - if (mockup is null) - { - continue; - } - - var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); - - var configPath = Path.Combine(userDataDir, "config.xml"); - var config = File.Exists(configPath) - ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() - : new UserConfiguration(); - - var policyPath = Path.Combine(userDataDir, "policy.xml"); - var policy = File.Exists(policyPath) - ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() - : new UserPolicy(); - policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( - "Emby.Server.Implementations.Library", - "Jellyfin.Server.Implementations.Users", - StringComparison.Ordinal) - ?? typeof(DefaultAuthenticationProvider).FullName; - - policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; - int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch - { - -1 => null, - 0 => 3, - _ => policy.LoginAttemptsBeforeLockout - }; + Id = entry.GetGuid(1), + InternalId = entry.GetInt64(0), + MaxParentalRatingScore = policy.MaxParentalRating, + MaxParentalRatingSubScore = null, + EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, + RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, + InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, + LoginAttemptsBeforeLockout = maxLoginAttempts, + SubtitleMode = config.SubtitleMode, + HidePlayedInLatest = config.HidePlayedInLatest, + EnableLocalPassword = config.EnableLocalPassword, + PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, + DisplayCollectionsView = config.DisplayCollectionsView, + DisplayMissingEpisodes = config.DisplayMissingEpisodes, + AudioLanguagePreference = config.AudioLanguagePreference, + RememberAudioSelections = config.RememberAudioSelections, + EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, + RememberSubtitleSelections = config.RememberSubtitleSelections, + SubtitleLanguagePreference = config.SubtitleLanguagePreference, + Password = mockup.Password, + LastLoginDate = mockup.LastLoginDate, + LastActivityDate = mockup.LastActivityDate + }; + + if (mockup.ImageInfos.Length > 0) + { + ItemImageInfo info = mockup.ImageInfos[0]; - var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) + user.ProfileImage = new ImageInfo(info.Path) { - Id = entry.GetGuid(1), - InternalId = entry.GetInt64(0), - MaxParentalAgeRating = policy.MaxParentalRating, - EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, - RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, - InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, - LoginAttemptsBeforeLockout = maxLoginAttempts, - SubtitleMode = config.SubtitleMode, - HidePlayedInLatest = config.HidePlayedInLatest, - EnableLocalPassword = config.EnableLocalPassword, - PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, - DisplayCollectionsView = config.DisplayCollectionsView, - DisplayMissingEpisodes = config.DisplayMissingEpisodes, - AudioLanguagePreference = config.AudioLanguagePreference, - RememberAudioSelections = config.RememberAudioSelections, - EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, - RememberSubtitleSelections = config.RememberSubtitleSelections, - SubtitleLanguagePreference = config.SubtitleLanguagePreference, - Password = mockup.Password, - LastLoginDate = mockup.LastLoginDate, - LastActivityDate = mockup.LastActivityDate + LastModified = info.DateModified }; + } - if (mockup.ImageInfos.Length > 0) - { - ItemImageInfo info = mockup.ImageInfos[0]; - - user.ProfileImage = new ImageInfo(info.Path) - { - LastModified = info.DateModified - }; - } - - user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); - user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); - user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); - user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); - user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); - user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); - user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); - user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); - user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); - user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); - user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); - user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); - user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); - user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); - user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); - user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); - user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); - user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); - user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); - user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); - - foreach (var policyAccessSchedule in policy.AccessSchedules) - { - user.AccessSchedules.Add(policyAccessSchedule); - } - - user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); - user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); - user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); - user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); - user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); - user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); - user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); - user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); - user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); - - dbContext.Users.Add(user); + user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); + user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); + user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); + user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); + user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); + user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); + user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); + user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); + user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); + user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); + user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); + user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); + user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); + user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); + user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); + user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); + user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); + user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); + user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); + user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); + + foreach (var policyAccessSchedule in policy.AccessSchedules) + { + user.AccessSchedules.Add(policyAccessSchedule); } - dbContext.SaveChanges(); + user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); + user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); + user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); + user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); + user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); + user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); + user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); + user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); + user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); + + dbContext.Users.Add(user); } - try - { - File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + dbContext.SaveChanges(); + } - var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); - if (File.Exists(journalPath)) - { - File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); - } - } - catch (IOException e) + try + { + File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + + var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); + if (File.Exists(journalPath)) { - _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); } } + catch (IOException e) + { + _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + } + } #nullable disable - internal class UserMockup - { - public string Password { get; set; } + internal class UserMockup + { + public string Password { get; set; } - public string EasyPassword { get; set; } + public string EasyPassword { get; set; } - public DateTime? LastLoginDate { get; set; } + public DateTime? LastLoginDate { get; set; } - public DateTime? LastActivityDate { get; set; } + public DateTime? LastActivityDate { get; set; } - public string Name { get; set; } + public string Name { get; set; } - public ItemImageInfo[] ImageInfos { get; set; } - } + public ItemImageInfo[] ImageInfos { get; set; } } } diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs new file mode 100644 index 000000000..8b394dd7a --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -0,0 +1,295 @@ +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-20T21:00:00", nameof(MoveExtractedFiles))] +public class MoveExtractedFiles : IAsyncMigrationRoutine +{ + private readonly IApplicationPaths _appPaths; + private readonly ILogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IPathManager _pathManager; + private readonly IFileSystem _fileSystem; + + /// <summary> + /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class. + /// </summary> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + public MoveExtractedFiles( + IApplicationPaths appPaths, + ILogger<MoveExtractedFiles> logger, + IStartupLogger<MoveExtractedFiles> startupLogger, + IPathManager pathManager, + IFileSystem fileSystem, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _appPaths = appPaths; + _logger = startupLogger.With(logger); + _pathManager = pathManager; + _fileSystem = fileSystem; + _dbProvider = dbProvider; + } + + private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles"); + + private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); + + /// <inheritdoc /> + public async Task PerformAsync(CancellationToken cancellationToken) + { + const int Limit = 5000; + int itemCount = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder); + _logger.LogInformation("Checking {Count} items for movable extracted files.", records); + + // Make sure directories exist + Directory.CreateDirectory(SubtitleCachePath); + Directory.CreateDirectory(AttachmentCachePath); + + await foreach (var result in context.BaseItems + .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal)) + .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder) + .Select(b => new + { + b.Id, + b.Path, + b.MediaStreams + }) + .OrderBy(e => e.Id) + .WithPartitionProgress((partition) => _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", partition * Limit, itemCount, sw.Elapsed)) + .PartitionEagerAsync(Limit, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + if (MoveSubtitleAndAttachmentFiles(result.Id, result.Path, result.MediaStreams, context)) + { + itemCount++; + } + } + + _logger.LogInformation("Moved files for {Count} items in {Time}", itemCount, sw.Elapsed); + + // Get all subdirectories with 1 character names (those are the legacy directories) + var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList(); + subdirectories.AddRange(Directory.GetDirectories(AttachmentCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == AttachmentCachePath.Length + 2)); + + // Remove all legacy subdirectories + foreach (var subdir in subdirectories) + { + Directory.Delete(subdir, true); + } + + // Remove old cache path + var attachmentCachePath = Path.Join(_appPaths.CachePath, "attachments"); + if (Directory.Exists(attachmentCachePath)) + { + Directory.Delete(attachmentCachePath, true); + } + + _logger.LogInformation("Cleaned up left over subtitles and attachments."); + } + + private bool MoveSubtitleAndAttachmentFiles(Guid id, string? path, ICollection<MediaStreamInfo>? mediaStreams, JellyfinDbContext context) + { + var itemIdString = id.ToString("N", CultureInfo.InvariantCulture); + var modified = false; + if (mediaStreams is not null) + { + foreach (var mediaStream in mediaStreams) + { + if (mediaStream.Codec is null) + { + continue; + } + + var mediaStreamIndex = mediaStream.StreamIndex; + var extension = GetSubtitleExtension(mediaStream.Codec); + var oldSubtitleCachePath = GetOldSubtitleCachePath(path, mediaStreamIndex, extension); + if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath)) + { + continue; + } + + var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension); + if (File.Exists(newSubtitleCachePath)) + { + File.Delete(oldSubtitleCachePath); + } + else + { + var newDirectory = Path.GetDirectoryName(newSubtitleCachePath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldSubtitleCachePath, newSubtitleCachePath, false); + _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, id, oldSubtitleCachePath, newSubtitleCachePath); + + modified = true; + } + } + } + } + +#pragma warning disable CA1309 // Use ordinal string comparison + var attachments = context.AttachmentStreamInfos.Where(a => a.ItemId.Equals(id) && !string.Equals(a.Codec, "mjpeg")).ToList(); +#pragma warning restore CA1309 // Use ordinal string comparison + var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.Filename) + && (a.Filename.Contains('/', StringComparison.OrdinalIgnoreCase) || a.Filename.Contains('\\', StringComparison.OrdinalIgnoreCase))); + foreach (var attachment in attachments) + { + var attachmentIndex = attachment.Index; + var oldAttachmentPath = GetOldAttachmentDataPath(path, attachmentIndex); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + continue; + } + } + + var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.Filename ?? attachmentIndex.ToString(CultureInfo.InvariantCulture)); + if (File.Exists(newAttachmentPath)) + { + File.Delete(oldAttachmentPath); + } + else + { + var newDirectory = Path.GetDirectoryName(newAttachmentPath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldAttachmentPath, newAttachmentPath, false); + _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, id, oldAttachmentPath, newAttachmentPath); + + modified = true; + } + } + } + + return modified; + } + + private string? GetOldAttachmentDataPath(string? mediaPath, int attachmentStreamIndex) + { + if (mediaPath is null) + { + return null; + } + + string filename; + if (_fileSystem.IsPathFile(mediaPath)) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(mediaPath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping attachment at index {Index} for {Path}: {Exception}", attachmentStreamIndex, mediaPath, e.Message); + + return null; + } + + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + else + { + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + + return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename); + } + + private string? GetOldAttachmentCachePath(string mediaSourceId, AttachmentStreamInfo attachment, bool shouldExtractOneByOne) + { + var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId); + if (shouldExtractOneByOne) + { + return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture)); + } + + if (string.IsNullOrEmpty(attachment.Filename)) + { + return null; + } + + return Path.Join(attachmentFolderPath, attachment.Filename); + } + + private string? GetOldSubtitleCachePath(string? path, int streamIndex, string outputSubtitleExtension) + { + if (path is null) + { + return null; + } + + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(path); + } + catch (IOException e) + { + _logger.LogDebug("Skipping subtitle at index {Index} for {Path}: {Exception}", streamIndex, path, e.Message); + + return null; + } + + var ticksParam = string.Empty; + ReadOnlySpan<char> filename = new Guid(MD5.HashData(Encoding.Unicode.GetBytes(path + "_" + streamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture) + ticksParam))) + outputSubtitleExtension; + + return Path.Join(SubtitleCachePath, filename[..1], filename); + } + + private static string GetSubtitleExtension(string codec) + { + if (codec.ToLower(CultureInfo.InvariantCulture).Equals("ass", StringComparison.OrdinalIgnoreCase) + || codec.ToLower(CultureInfo.InvariantCulture).Equals("ssa", StringComparison.OrdinalIgnoreCase)) + { + return "." + codec; + } + else if (codec.Contains("pgs", StringComparison.OrdinalIgnoreCase)) + { + return ".sup"; + } + else + { + return ".srt"; + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index c1a9e8894..0f55465e8 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,7 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; -using MediaBrowser.Common; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -16,12 +16,15 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to move trickplay files to the new directory. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T23:00:00", nameof(MoveTrickplayFiles), RunMigrationOnSetup = true)] public class MoveTrickplayFiles : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; private readonly ILibraryManager _libraryManager; - private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IStartupLogger _logger; /// <summary> /// Initializes a new instance of the <see cref="MoveTrickplayFiles"/> class. @@ -30,7 +33,11 @@ public class MoveTrickplayFiles : IMigrationRoutine /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> /// <param name="logger">The logger.</param> - public MoveTrickplayFiles(ITrickplayManager trickplayManager, IFileSystem fileSystem, ILibraryManager libraryManager, ILogger<MoveTrickplayFiles> logger) + public MoveTrickplayFiles( + ITrickplayManager trickplayManager, + IFileSystem fileSystem, + ILibraryManager libraryManager, + IStartupLogger<MoveTrickplayFiles> logger) { _trickplayManager = trickplayManager; _fileSystem = fileSystem; @@ -39,18 +46,9 @@ public class MoveTrickplayFiles : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("4EF123D5-8EFF-4B0B-869D-3AED07A60E1B"); - - /// <inheritdoc /> - public string Name => "MoveTrickplayFiles"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { - const int Limit = 100; + const int Limit = 5000; int itemCount = 0, offset = 0, previousCount; var sw = Stopwatch.StartNew(); @@ -65,9 +63,6 @@ public class MoveTrickplayFiles : IMigrationRoutine do { var trickplayInfos = _trickplayManager.GetTrickplayItemsAsync(Limit, offset).GetAwaiter().GetResult(); - previousCount = trickplayInfos.Count; - offset += Limit; - trickplayQuery.ItemIds = trickplayInfos.Select(i => i.ItemId).Distinct().ToArray(); var items = _libraryManager.GetItemList(trickplayQuery); foreach (var trickplayInfo in trickplayInfos) @@ -78,18 +73,32 @@ public class MoveTrickplayFiles : IMigrationRoutine continue; } - if (++itemCount % 1_000 == 0) + var moved = false; + var oldPath = GetOldTrickplayDirectory(item, trickplayInfo.Width); + var newPath = _trickplayManager.GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); + if (_fileSystem.DirectoryExists(oldPath)) { - _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); + _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; } - var oldPath = GetOldTrickplayDirectory(item, trickplayInfo.Width); - var newPath = _trickplayManager.GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); + oldPath = GetNewOldTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); if (_fileSystem.DirectoryExists(oldPath)) { _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; + } + + if (moved) + { + itemCount++; } } + + offset += Limit; + previousCount = trickplayInfos.Count; + + _logger.LogInformation("Checked: {Checked} - Moved: {Count} - Time: {Time}", offset, itemCount, sw.Elapsed); } while (previousCount == Limit); _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); @@ -101,4 +110,20 @@ public class MoveTrickplayFiles : IMigrationRoutine return width.HasValue ? Path.Combine(path, width.Value.ToString(CultureInfo.InvariantCulture)) : path; } + + private string GetNewOldTrickplayDirectory(BaseItem item, int tileWidth, int tileHeight, int width, bool saveWithMedia = false) + { + var path = saveWithMedia + ? Path.Combine(item.ContainingFolderPath, Path.ChangeExtension(item.Path, ".trickplay")) + : Path.Combine(item.GetInternalMetadataPath(), "trickplay"); + + var subdirectory = string.Format( + CultureInfo.InvariantCulture, + "{0} - {1}x{2}", + width.ToString(CultureInfo.InvariantCulture), + tileWidth.ToString(CultureInfo.InvariantCulture), + tileHeight.ToString(CultureInfo.InvariantCulture)); + + return Path.Combine(path, subdirectory); + } } diff --git a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs index 9cfaec46f..ebf4a2780 100644 --- a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs @@ -2,48 +2,41 @@ using System; using MediaBrowser.Controller.Configuration; using MediaBrowser.Model.Updates; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to initialize system configuration with the default plugin repository. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T11:00:00", nameof(ReaddDefaultPluginRepository), "5F86E7F6-D966-4C77-849D-7A7B40B68C4E", RunMigrationOnSetup = true)] +public class ReaddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private readonly IServerConfigurationManager _serverConfigurationManager; + + private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo + { + Name = "Jellyfin Stable", + Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" + }; + /// <summary> - /// Migration to initialize system configuration with the default plugin repository. + /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. /// </summary> - public class ReaddDefaultPluginRepository : IMigrationRoutine + /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) { - private readonly IServerConfigurationManager _serverConfigurationManager; - - private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo - { - Name = "Jellyfin Stable", - Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" - }; - - /// <summary> - /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. - /// </summary> - /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> - public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) - { - _serverConfigurationManager = serverConfigurationManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("5F86E7F6-D966-4C77-849D-7A7B40B68C4E"); - - /// <inheritdoc/> - public string Name => "ReaddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; + _serverConfigurationManager = serverConfigurationManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + // Only add if repository list is empty + if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) { - // Only add if repository list is empty - if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) - { - _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; - _serverConfigurationManager.SaveConfiguration(); - } + _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; + _serverConfigurationManager.SaveConfiguration(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs new file mode 100644 index 000000000..b23a7dbc4 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to re-read creation dates for library items with internal metadata paths. +/// </summary> +[JellyfinMigration("2025-04-20T23:00:00", nameof(RefreshInternalDateModified))] +public class RefreshInternalDateModified : IDatabaseMigrationRoutine +{ + private readonly ILogger<RefreshInternalDateModified> _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IFileSystem _fileSystem; + private readonly IServerApplicationHost _applicationHost; + private readonly bool _useFileCreationTimeForDateAdded; + + private IReadOnlyList<string> _internalTypes = [ + typeof(Genre).FullName!, + typeof(MusicGenre).FullName!, + typeof(MusicArtist).FullName!, + typeof(People).FullName!, + typeof(Studio).FullName! + ]; + + private IReadOnlyList<string> _internalPaths; + + /// <summary> + /// Initializes a new instance of the <see cref="RefreshInternalDateModified"/> class. + /// </summary> + /// <param name="applicationHost">Instance of the <see cref="IServerApplicationHost"/> interface.</param> + /// <param name="applicationPaths">Instance of the <see cref="IServerApplicationPaths"/> interface.</param> + /// <param name="configurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + public RefreshInternalDateModified( + IServerApplicationHost applicationHost, + IServerApplicationPaths applicationPaths, + IServerConfigurationManager configurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + ILogger<RefreshInternalDateModified> logger, + IFileSystem fileSystem) + { + _dbProvider = dbProvider; + _logger = logger; + _fileSystem = fileSystem; + _applicationHost = applicationHost; + _internalPaths = [ + applicationPaths.ArtistsPath, + applicationPaths.GenrePath, + applicationPaths.MusicGenrePath, + applicationPaths.StudioPath, + applicationPaths.PeoplePath + ]; + _useFileCreationTimeForDateAdded = configurationManager.GetMetadataConfiguration().UseFileCreationTimeForDateAdded; + } + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => _internalTypes.Contains(b.Type)); + _logger.LogInformation("Checking if {Count} potentially internal items require refreshed DateModified", records); + + do + { + var results = context.BaseItems + .Where(b => _internalTypes.Contains(b.Type)) + .OrderBy(e => e.Id) + .Skip(offset) + .Take(Limit) + .ToList(); + + foreach (var item in results) + { + var itemPath = item.Path; + if (itemPath is not null) + { + var realPath = _applicationHost.ExpandVirtualPath(item.Path); + if (_internalPaths.Any(path => realPath.StartsWith(path, StringComparison.Ordinal))) + { + var writeTime = _fileSystem.GetLastWriteTimeUtc(realPath); + var itemModificationTime = item.DateModified; + if (writeTime != itemModificationTime) + { + _logger.LogDebug("Reset file modification date: Old: {Old} - New: {New} - Path: {Path}", itemModificationTime, writeTime, realPath); + item.DateModified = writeTime; + if (_useFileCreationTimeForDateAdded) + { + item.DateCreated = _fileSystem.GetCreationTimeUtc(realPath); + } + + itemCount++; + } + } + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Refreshed: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + + _logger.LogInformation("Refreshed DateModified for {Count} items in {Time}", itemCount, sw.Elapsed); + } +} diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs index 52fb93d59..b626c473e 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs @@ -3,50 +3,43 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Removes the old 'RemoveDownloadImagesInAdvance' from library options. - /// </summary> - internal class RemoveDownloadImagesInAdvance : IMigrationRoutine - { - private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; - private readonly ILibraryManager _libraryManager; - - public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) - { - _logger = logger; - _libraryManager = libraryManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{A81F75E0-8F43-416F-A5E8-516CCAB4D8CC}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "RemoveDownloadImagesInAdvance"; +/// <summary> +/// Removes the old 'RemoveDownloadImagesInAdvance' from library options. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T13:00:00", nameof(RemoveDownloadImagesInAdvance), "A81F75E0-8F43-416F-A5E8-516CCAB4D8CC")] +internal class RemoveDownloadImagesInAdvance : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete +{ + private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; + private readonly ILibraryManager _libraryManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) + { + _logger = logger; + _libraryManager = libraryManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + var virtualFolders = _libraryManager.GetVirtualFolders(false); + _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); + foreach (var virtualFolder in virtualFolders) { - var virtualFolders = _libraryManager.GetVirtualFolders(false); - _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); - foreach (var virtualFolder in virtualFolders) + // Some virtual folders don't have a proper item id. + if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) { - // Some virtual folders don't have a proper item id. - if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) - { - continue; - } - - var libraryOptions = virtualFolder.LibraryOptions; - var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); - // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. - collectionFolder.UpdateLibraryOptions(libraryOptions); - _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); + continue; } + + var libraryOptions = virtualFolder.LibraryOptions; + var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); + // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. + collectionFolder.UpdateLibraryOptions(libraryOptions); + _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs index 7b0d9456d..c9e66d0cf 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs @@ -7,77 +7,70 @@ using MediaBrowser.Controller; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T08:00:00", nameof(RemoveDuplicateExtras), "ACBE17B7-8435-4A83-8B64-6FCF162CB9BD")] +internal class RemoveDuplicateExtras : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - /// <summary> - /// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. - /// </summary> - internal class RemoveDuplicateExtras : IMigrationRoutine + private const string DbFilename = "library.db"; + private readonly ILogger<RemoveDuplicateExtras> _logger; + private readonly IServerApplicationPaths _paths; + + public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) { - private const string DbFilename = "library.db"; - private readonly ILogger<RemoveDuplicateExtras> _logger; - private readonly IServerApplicationPaths _paths; + _logger = logger; + _paths = paths; + } - public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + var dbPath = Path.Combine(dataPath, DbFilename); + using var connection = new SqliteConnection($"Filename={dbPath}"); + connection.Open(); + using (var transaction = connection.BeginTransaction()) { - _logger = logger; - _paths = paths; - } + // Query the database for the ids of duplicate extras + var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); + var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - /// <inheritdoc/> - public Guid Id => Guid.Parse("{ACBE17B7-8435-4A83-8B64-6FCF162CB9BD}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicateExtras"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - var dbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + // Do nothing if no duplicate extras were detected + if (bads.Length == 0) { - // Query the database for the ids of duplicate extras - var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); - var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - - // Do nothing if no duplicate extras were detected - if (bads.Length == 0) - { - _logger.LogInformation("No duplicate extras detected, skipping migration."); - return; - } + _logger.LogInformation("No duplicate extras detected, skipping migration."); + return; + } - // Back up the database before deleting any entries - for (int i = 1; ; i++) + // Back up the database before deleting any entries + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); + if (!File.Exists(bakPath)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + try { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } + File.Copy(dbPath, bakPath); + _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; } } - - // Delete all duplicate extras - _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); - connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); - transaction.Commit(); } + + // Delete all duplicate extras + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); + connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); + transaction.Commit(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs index f84bccc25..23f212424 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs @@ -1,44 +1,33 @@ using System; using System.Linq; using System.Threading; - using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Playlists; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Remove duplicate playlist entries. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T19:00:00", nameof(RemoveDuplicatePlaylistChildren), "96C156A2-7A13-4B3B-A8B8-FB80C94D20C0")] internal class RemoveDuplicatePlaylistChildren : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private readonly ILogger<RemoveDuplicatePlaylistChildren> _logger; private readonly ILibraryManager _libraryManager; private readonly IPlaylistManager _playlistManager; public RemoveDuplicatePlaylistChildren( - ILogger<RemoveDuplicatePlaylistChildren> logger, ILibraryManager libraryManager, IPlaylistManager playlistManager) { - _logger = logger; _libraryManager = libraryManager; _playlistManager = playlistManager; } /// <inheritdoc/> - public Guid Id => Guid.Parse("{96C156A2-7A13-4B3B-A8B8-FB80C94D20C0}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicatePlaylistChildren"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/ReseedFolderFlag.cs b/Jellyfin.Server/Migrations/Routines/ReseedFolderFlag.cs new file mode 100644 index 000000000..502763ac0 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/ReseedFolderFlag.cs @@ -0,0 +1,74 @@ +#pragma warning disable RS0030 // Do not use banned APIs +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Emby.Server.Implementations.Data; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +[JellyfinMigration("2025-07-30T21:50:00", nameof(ReseedFolderFlag))] +[JellyfinMigrationBackup(JellyfinDb = true)] +internal class ReseedFolderFlag : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db.old"; + + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + + public ReseedFolderFlag( + IStartupLogger<MigrateLibraryDb> startupLogger, + IDbContextFactory<JellyfinDbContext> provider, + IServerApplicationPaths paths) + { + _logger = startupLogger; + _provider = provider; + _paths = paths; + } + + internal static bool RerunGuardFlag { get; set; } = false; + + public async Task PerformAsync(CancellationToken cancellationToken) + { + if (RerunGuardFlag) + { + _logger.LogInformation("Migration is skipped because it does not apply."); + return; + } + + _logger.LogInformation("Migrating the IsFolder flag from library.db.old may take a while, do not stop Jellyfin."); + + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate IsFolder flag from {LibraryDb} as it does not exist. This migration expects the MigrateLibraryDb to run first.", libraryDbPath); + return; + } + + var dbContext = await _provider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + var queryResult = connection.Query( + """ + SELECT guid FROM TypedBaseItems + WHERE IsFolder = true + """) + .Select(entity => entity.GetGuid(0)) + .ToList(); + _logger.LogInformation("Migrating the IsFolder flag for {Count} items.", queryResult.Count); + foreach (var id in queryResult) + { + await dbContext.BaseItems.Where(e => e.Id == id).ExecuteUpdateAsync(e => e.SetProperty(f => f.IsFolder, true), cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs index 7e8c8ac87..f58cf2741 100644 --- a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs @@ -6,7 +6,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to update the default Jellyfin plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T17:00:00", nameof(UpdateDefaultPluginRepository), "852816E0-2712-49A9-9240-C6FC5FCAD1A8", RunMigrationOnSetup = true)] public class UpdateDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string NewRepositoryUrl = "https://repo.jellyfin.org/files/plugin/manifest.json"; private const string OldRepositoryUrl = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json"; @@ -23,15 +26,6 @@ public class UpdateDefaultPluginRepository : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("852816E0-2712-49A9-9240-C6FC5FCAD1A8"); - - /// <inheritdoc /> - public string Name => "UpdateDefaultPluginRepository10.9"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { var updated = false; |