diff options
Diffstat (limited to 'Jellyfin.Server/Migrations/Routines')
22 files changed, 1419 insertions, 739 deletions
diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs index 2047ec743..00d152b4b 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultCastReceivers.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to add the default cast receivers to the system config. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T16:00:00", nameof(AddDefaultCastReceivers), "34A1A1C4-5572-418E-A2F8-32CDFE2668E8", RunMigrationOnSetup = true)] public class AddDefaultCastReceivers : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -21,15 +24,6 @@ public class AddDefaultCastReceivers : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("34A1A1C4-5572-418E-A2F8-32CDFE2668E8"); - - /// <inheritdoc /> - public string Name => "AddDefaultCastReceivers"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { _serverConfigurationManager.Configuration.CastReceiverApplications = diff --git a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs index fc6b5d597..8c8398a16 100644 --- a/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/AddDefaultPluginRepository.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Migration to initialize system configuration with the default plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T09:00:00", nameof(AddDefaultPluginRepository), "EB58EBEE-9514-4B9B-8225-12E1A40020DF", RunMigrationOnSetup = true)] public class AddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly IServerConfigurationManager _serverConfigurationManager; @@ -27,15 +30,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("EB58EBEE-9514-4B9B-8225-12E1A40020DF"); - - /// <inheritdoc/> - public string Name => "AddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; - - /// <inheritdoc/> public void Perform() { _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; diff --git a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs index 5a8ef2e1c..1326a6dc8 100644 --- a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs +++ b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs @@ -12,7 +12,10 @@ namespace Jellyfin.Server.Migrations.Routines /// If the deprecated logging.json file exists and has a custom config, it will be used as logging.user.json, /// otherwise a blank file will be created. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T06:00:00", nameof(CreateUserLoggingConfigFile), "EF103419-8451-40D8-9F34-D1A8E93A1679")] internal class CreateUserLoggingConfigFile : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { /// <summary> /// File history for logging.json as existed during this migration creation. The contents for each has been minified. @@ -43,15 +46,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{EF103419-8451-40D8-9F34-D1A8E93A1679}"); - - /// <inheritdoc/> - public string Name => "CreateLoggingConfigHierarchy"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logDirectory = _appPaths.ConfigurationDirectoryPath; diff --git a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs index 378e88e25..acf2835fe 100644 --- a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs +++ b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs @@ -7,7 +7,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Disable transcode throttling for all installations since it is currently broken for certain video formats. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T05:00:00", nameof(DisableTranscodingThrottling), "4124C2CD-E939-4FFB-9BE9-9B311C413638")] internal class DisableTranscodingThrottling : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<DisableTranscodingThrottling> _logger; private readonly IConfigurationManager _configManager; @@ -19,15 +22,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{4124C2CD-E939-4FFB-9BE9-9B311C413638}"); - - /// <inheritdoc/> - public string Name => "DisableTranscodingThrottling"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { // Set EnableThrottling to false since it wasn't used before and may introduce issues diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index a20253369..05ded06ba 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -16,9 +16,12 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// Fixes the data column of audio types to be deserializable. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T18:00:00", nameof(FixAudioData), "CF6FABC2-9FBE-4933-84A5-FFE52EF22A58")] + [JellyfinMigrationBackup(LegacyLibraryDb = true)] internal class FixAudioData : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private const string DbFilename = "library.db"; private readonly ILogger<FixAudioData> _logger; private readonly IServerApplicationPaths _applicationPaths; private readonly IItemRepository _itemRepository; @@ -34,40 +37,8 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("{CF6FABC2-9FBE-4933-84A5-FFE52EF22A58}"); - - /// <inheritdoc/> - public string Name => "FixAudioData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) - { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); - File.Copy(dbPath, bakPath); - _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } - } - _logger.LogInformation("Backfilling audio lyrics data to database."); var startIndex = 0; var records = _itemRepository.GetCount(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs index 192c170b2..56614ece3 100644 --- a/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs +++ b/Jellyfin.Server/Migrations/Routines/FixPlaylistOwner.cs @@ -13,7 +13,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Properly set playlist owner. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T15:00:00", nameof(FixPlaylistOwner), "615DFA9E-2497-4DBB-A472-61938B752C5B")] internal class FixPlaylistOwner : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ILogger<FixPlaylistOwner> _logger; private readonly ILibraryManager _libraryManager; @@ -30,15 +33,6 @@ internal class FixPlaylistOwner : IMigrationRoutine } /// <inheritdoc/> - public Guid Id => Guid.Parse("{615DFA9E-2497-4DBB-A472-61938B752C5B}"); - - /// <inheritdoc/> - public string Name => "FixPlaylistOwner"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs index e9fe9abce..a954d307e 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateActivityLogDb.cs @@ -14,7 +14,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the activity log database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T07:00:00", nameof(MigrateActivityLogDb), "3793eb59-bc8c-456c-8b9f-bd5a62a42978")] public class MigrateActivityLogDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "activitylog.db"; @@ -36,15 +39,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc/> - public Guid Id => Guid.Parse("3793eb59-bc8c-456c-8b9f-bd5a62a42978"); - - /// <inheritdoc/> - public string Name => "MigrateActivityLogDatabase"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var logLevelDictionary = new Dictionary<string, LogLevel>(StringComparer.OrdinalIgnoreCase) diff --git a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs index feaf46c84..c6699c21d 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateAuthenticationDb.cs @@ -15,7 +15,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// A migration that moves data from the authentication database into the new schema. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T14:00:00", nameof(MigrateAuthenticationDb), "5BD72F41-E6F3-4F60-90AA-09869ABE0E22")] public class MigrateAuthenticationDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "authentication.db"; @@ -44,15 +47,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("5BD72F41-E6F3-4F60-90AA-09869ABE0E22"); - - /// <inheritdoc /> - public string Name => "MigrateAuthenticationDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { var dataPath = _appPaths.DataPath; diff --git a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs index a8fa2e52a..0d9952ce9 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateDisplayPreferencesDb.cs @@ -20,7 +20,10 @@ namespace Jellyfin.Server.Migrations.Routines /// <summary> /// The migration routine for migrating the display preferences database to EF Core. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete + [JellyfinMigration("2025-04-20T12:00:00", nameof(MigrateDisplayPreferencesDb), "06387815-C3CC-421F-A888-FB5F9992BEA8")] public class MigrateDisplayPreferencesDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string DbFilename = "displaypreferences.db"; @@ -52,15 +55,6 @@ namespace Jellyfin.Server.Migrations.Routines } /// <inheritdoc /> - public Guid Id => Guid.Parse("06387815-C3CC-421F-A888-FB5F9992BEA8"); - - /// <inheritdoc /> - public string Name => "MigrateDisplayPreferencesDatabase"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => false; - - /// <inheritdoc /> public void Perform() { HomeSectionType[] defaults = diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs new file mode 100644 index 000000000..033045e63 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -0,0 +1,152 @@ +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions.Json; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-21T00:00:00", nameof(MigrateKeyframeData))] +public class MigrateKeyframeData : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IApplicationPaths _appPaths; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="dbProvider">The EFCore db factory.</param> + public MigrateKeyframeData( + IStartupLogger startupLogger, + IApplicationPaths appPaths, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _logger = startupLogger; + _appPaths = appPaths; + _dbProvider = dbProvider; + } + + private string KeyframeCachePath => Path.Combine(_appPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var baseQuery = context.BaseItems.Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder).OrderBy(e => e.Id); + var records = baseQuery.Count(); + _logger.LogInformation("Checking {Count} items for importable keyframe data.", records); + + context.KeyframeData.ExecuteDelete(); + using var transaction = context.Database.BeginTransaction(); + do + { + var results = baseQuery.Skip(offset).Take(Limit).Select(b => new Tuple<Guid, string?>(b.Id, b.Path)).ToList(); + foreach (var result in results) + { + if (TryGetKeyframeData(result.Item1, result.Item2, out var data)) + { + itemCount++; + context.KeyframeData.Add(data); + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Imported: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + transaction.Commit(); + + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + + if (Directory.Exists(KeyframeCachePath)) + { + Directory.Delete(KeyframeCachePath, true); + } + } + + private bool TryGetKeyframeData(Guid id, string? path, [NotNullWhen(true)] out KeyframeData? data) + { + data = null; + if (!string.IsNullOrEmpty(path)) + { + var cachePath = GetCachePath(KeyframeCachePath, path); + if (TryReadFromCache(cachePath, out var keyframeData)) + { + data = new() + { + ItemId = id, + KeyframeTicks = keyframeData.KeyframeTicks.ToList(), + TotalDuration = keyframeData.TotalDuration + }; + + return true; + } + } + + return false; + } + + private string? GetCachePath(string keyframeCachePath, string filePath) + { + DateTime? lastWriteTimeUtc; + try + { + lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping {Path}: {Exception}", filePath, e.Message); + + return null; + } + + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string? cachePath, [NotNullWhen(true)] out MediaEncoding.Keyframes.KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<MediaEncoding.Keyframes.KeyframeData>(bytes, _jsonOptions); + + return cachedResult is not null; + } + + cachedResult = null; + + return false; + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index 427f04f9d..521655a4f 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -9,12 +9,12 @@ using System.Globalization; using System.IO; using System.Linq; using System.Text; -using System.Threading; using Emby.Server.Implementations.Data; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Extensions; using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; @@ -29,11 +29,13 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// The migration routine for migrating the userdata database to EF Core. /// </summary> +[JellyfinMigration("2025-04-20T20:00:00", nameof(MigrateLibraryDb))] +[JellyfinMigrationBackup(JellyfinDb = true, LegacyLibraryDb = true)] internal class MigrateLibraryDb : IDatabaseMigrationRoutine { private const string DbFilename = "library.db"; - private readonly ILogger<MigrateLibraryDb> _logger; + private readonly IStartupLogger _logger; private readonly IServerApplicationPaths _paths; private readonly IJellyfinDatabaseProvider _jellyfinDatabaseProvider; private readonly IDbContextFactory<JellyfinDbContext> _provider; @@ -41,305 +43,379 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine /// <summary> /// Initializes a new instance of the <see cref="MigrateLibraryDb"/> class. /// </summary> - /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> /// <param name="provider">The database provider.</param> /// <param name="paths">The server application paths.</param> /// <param name="jellyfinDatabaseProvider">The database provider for special access.</param> public MigrateLibraryDb( - ILogger<MigrateLibraryDb> logger, + IStartupLogger startupLogger, IDbContextFactory<JellyfinDbContext> provider, IServerApplicationPaths paths, IJellyfinDatabaseProvider jellyfinDatabaseProvider) { - _logger = logger; + _logger = startupLogger; _provider = provider; _paths = paths; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; } /// <inheritdoc/> - public Guid Id => Guid.Parse("36445464-849f-429f-9ad0-bb130efa0664"); - - /// <inheritdoc/> - public string Name => "MigrateLibraryDbData"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; // TODO Change back after testing - - /// <inheritdoc/> public void Perform() { _logger.LogInformation("Migrating the userdata from library.db may take a while, do not stop Jellyfin."); var dataPath = _paths.DataPath; var libraryDbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={libraryDbPath}"); - var migrationTotalTime = TimeSpan.Zero; + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } - var stopwatch = new Stopwatch(); - stopwatch.Start(); + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving TypedBaseItem."); - const string typedBaseItemsQuery = """ - SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, - IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, - PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, - ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, - Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, - DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, - PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, - ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems - """; - dbContext.BaseItems.ExecuteDelete(); + var fullOperationTimer = new Stopwatch(); + fullOperationTimer.Start(); - var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); - foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + using (var operation = GetPreparedDbContext("Cleanup database")) { - var baseItem = GetItem(dto); - dbContext.BaseItems.Add(baseItem.BaseItem); - foreach (var dataKey in baseItem.LegacyUserDataKey) - { - legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; - } + operation.JellyfinDbContext.AttachmentStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.BaseItems.ExecuteDelete(); + operation.JellyfinDbContext.ItemValues.ExecuteDelete(); + operation.JellyfinDbContext.UserData.ExecuteDelete(); + operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.Peoples.ExecuteDelete(); + operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete(); + operation.JellyfinDbContext.Chapters.ExecuteDelete(); + operation.JellyfinDbContext.AncestorIds.ExecuteDelete(); } - _logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); + connection.Open(); - _logger.LogInformation("Start moving ItemValues."); - // do not migrate inherited types as they are now properly mapped in search and lookup. - const string itemValueQuery = - """ - SELECT ItemId, Type, Value, CleanValue FROM ItemValues - WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) - """; - dbContext.ItemValues.ExecuteDelete(); + var baseItemIds = new HashSet<Guid>(); + using (var operation = GetPreparedDbContext("moving TypedBaseItem")) + { + const string typedBaseItemsQuery = + """ + SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, + IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, + PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, + ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, + Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, + DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, + PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, + ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems + """; + using (new TrackedMigrationStep("Loading TypedBaseItems", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + { + var baseItem = GetItem(dto); + operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem); + baseItemIds.Add(baseItem.BaseItem.Id); + foreach (var dataKey in baseItem.LegacyUserDataKey) + { + legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + } + } + } - // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. - var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } - foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + using (var operation = GetPreparedDbContext("moving ItemValues")) { - var itemId = dto.GetGuid(0); - var entity = GetItemValue(dto); - var key = ((int)entity.Type, entity.CleanValue); - if (!localItems.TryGetValue(key, out var existing)) + // do not migrate inherited types as they are now properly mapped in search and lookup. + const string itemValueQuery = + """ + SELECT ItemId, Type, Value, CleanValue FROM ItemValues + WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) + """; + + // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. + var localItems = new Dictionary<(int Type, string Value), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep("loading ItemValues", _logger)) { - localItems[key] = existing = (entity, []); + foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + { + var itemId = dto.GetGuid(0); + var entity = GetItemValue(dto); + var key = ((int)entity.Type, entity.Value); + if (!localItems.TryGetValue(key, out var existing)) + { + localItems[key] = existing = (entity, []); + } + + existing.ItemIds.Add(itemId); + } + + foreach (var item in localItems) + { + operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue); + operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + { + Item = null!, + ItemValue = null!, + ItemId = f, + ItemValueId = item.Value.ItemValue.ItemValueId + })); + } } - existing.ItemIds.Add(itemId); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - foreach (var item in localItems) + using (var operation = GetPreparedDbContext("moving UserData")) { - dbContext.ItemValues.Add(item.Value.ItemValue); - dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + var queryResult = connection.Query( + """ + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) + """); + + using (new TrackedMigrationStep("loading UserData", _logger)) { - Item = null!, - ItemValue = null!, - ItemId = f, - ItemValueId = item.Value.ItemValue.ItemValueId - })); - } + var users = operation.JellyfinDbContext.Users.AsNoTracking().ToImmutableArray(); + var userIdBlacklist = new HashSet<int>(); - _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + foreach (var entity in queryResult) + { + var userData = GetUserData(users, entity, userIdBlacklist); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); - _logger.LogInformation("Start moving UserData."); - var queryResult = connection.Query(""" - SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) - """); + continue; + } - dbContext.UserData.ExecuteDelete(); + if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + { + _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); + continue; + } - var users = dbContext.Users.AsNoTracking().ToImmutableArray(); + userData.ItemId = refItem.Id; + operation.JellyfinDbContext.UserData.Add(userData); + } - foreach (var entity in queryResult) - { - var userData = GetUserData(users, entity); - if (userData is null) - { - _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0)); - continue; + users.Clear(); } - if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + legacyBaseItemWithUserKeys.Clear(); + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger)) { - _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); - continue; + operation.JellyfinDbContext.SaveChanges(); } - - userData.ItemId = refItem.Id; - dbContext.UserData.Add(userData); } - users.Clear(); - legacyBaseItemWithUserKeys.Clear(); - _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count); - dbContext.SaveChanges(); - - _logger.LogInformation("Start moving MediaStreamInfos."); - const string mediaStreamQuery = """ - SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, - IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, - AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, - Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, - DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired - FROM MediaStreams - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) - """; - dbContext.MediaStreamInfos.ExecuteDelete(); - - foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + using (var operation = GetPreparedDbContext("moving MediaStreamInfos")) { - dbContext.MediaStreamInfos.Add(GetMediaStream(dto)); - } + const string mediaStreamQuery = + """ + SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, + IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, + AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, + Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, + DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired + FROM MediaStreams + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) + """; - _logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving People."); - const string personsQuery = """ - SELECT ItemId, Name, Role, PersonType, SortOrder FROM People - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) - """; - dbContext.Peoples.ExecuteDelete(); - dbContext.PeopleBaseItemMap.ExecuteDelete(); - - var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - var baseItemIds = dbContext.BaseItems.Select(b => b.Id).ToHashSet(); + using (new TrackedMigrationStep("loading MediaStreamInfos", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + { + operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto)); + } + } - foreach (SqliteDataReader reader in connection.Query(personsQuery)) - { - var itemId = reader.GetGuid(0); - if (!baseItemIds.Contains(itemId)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger)) { - _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); - continue; + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving AttachmentStreamInfos")) + { + const string mediaAttachmentQuery = + """ + SELECT ItemId, AttachmentIndex, Codec, CodecTag, Comment, filename, MIMEType + FROM mediaattachments + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = mediaattachments.ItemId) + """; - var entity = GetPerson(reader); - if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + using (new TrackedMigrationStep("loading AttachmentStreamInfos", _logger)) { - peopleCache[entity.Name] = personCache = (entity, []); + foreach (SqliteDataReader dto in connection.Query(mediaAttachmentQuery)) + { + operation.JellyfinDbContext.AttachmentStreamInfos.Add(GetMediaAttachment(dto)); + } } - if (reader.TryGetString(2, out var role)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AttachmentStreamInfos.Local.Count} AttachmentStreamInfos entries", _logger)) { + operation.JellyfinDbContext.SaveChanges(); } + } - int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); + using (var operation = GetPreparedDbContext("moving People")) + { + const string personsQuery = + """ + SELECT ItemId, Name, Role, PersonType, SortOrder FROM People + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) + """; - personCache.Items.Add(new PeopleBaseItemMap() + var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); + + using (new TrackedMigrationStep("loading People", _logger)) { - Item = null!, - ItemId = itemId, - People = null!, - PeopleId = personCache.Person.Id, - ListOrder = sortOrder, - SortOrder = sortOrder, - Role = role - }); - } + foreach (SqliteDataReader reader in connection.Query(personsQuery)) + { + var itemId = reader.GetGuid(0); + if (!baseItemIds.Contains(itemId)) + { + _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); + continue; + } - baseItemIds.Clear(); + var entity = GetPerson(reader); + if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + { + peopleCache[entity.Name] = personCache = (entity, []); + } - foreach (var item in peopleCache) - { - dbContext.Peoples.Add(item.Value.Person); - dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); - } + if (reader.TryGetString(2, out var role)) + { + } - peopleCache.Clear(); + int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); - _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + personCache.Items.Add(new PeopleBaseItemMap() + { + Item = null!, + ItemId = itemId, + People = null!, + PeopleId = personCache.Person.Id, + ListOrder = sortOrder, + SortOrder = sortOrder, + Role = role + }); + } - _logger.LogInformation("Start moving Chapters."); - const string chapterQuery = """ - SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) - """; - dbContext.Chapters.ExecuteDelete(); + baseItemIds.Clear(); - foreach (SqliteDataReader dto in connection.Query(chapterQuery)) - { - var chapter = GetChapter(dto); - dbContext.Chapters.Add(chapter); - } + foreach (var item in peopleCache) + { + operation.JellyfinDbContext.Peoples.Add(item.Value.Person); + operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); + } - _logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + peopleCache.Clear(); + } - _logger.LogInformation("Start moving AncestorIds."); - const string ancestorIdsQuery = """ - SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds - WHERE - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) - AND - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) - """; - dbContext.Chapters.ExecuteDelete(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } - foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + using (var operation = GetPreparedDbContext("moving Chapters")) { - var ancestorId = GetAncestorId(dto); - dbContext.AncestorIds.Add(ancestorId); + const string chapterQuery = + """ + SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) + """; + + using (new TrackedMigrationStep("loading Chapters", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(chapterQuery)) + { + var chapter = GetChapter(dto); + operation.JellyfinDbContext.Chapters.Add(chapter); + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.Chapters.Local.Count); + using (var operation = GetPreparedDbContext("moving AncestorIds")) + { + const string ancestorIdsQuery = + """ + SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds + WHERE + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) + AND + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) + """; - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (new TrackedMigrationStep("loading AncestorIds", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + { + var ancestorId = GetAncestorId(dto); + operation.JellyfinDbContext.AncestorIds.Add(ancestorId); + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } connection.Close(); + _logger.LogInformation("Migration of the Library.db done."); - _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + _logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed); SqliteConnection.ClearAllPools(); + _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); File.Move(libraryDbPath, libraryDbPath + ".old", true); + } - _logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime); - - _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); + private DatabaseMigrationStep GetPreparedDbContext(string operationName) + { + var dbContext = _provider.CreateDbContext(); + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return new DatabaseMigrationStep(dbContext, operationName, _logger); } - private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto) + private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto, HashSet<int> userIdBlacklist) { var internalUserId = dto.GetInt32(1); var user = users.FirstOrDefault(e => e.InternalId == internalUserId); if (user is null) { + if (userIdBlacklist.Contains(internalUserId)) + { + return null; + } + _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); return null; } @@ -654,6 +730,48 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return item; } + /// <summary> + /// Gets the attachment. + /// </summary> + /// <param name="reader">The reader.</param> + /// <returns>MediaAttachment.</returns> + private AttachmentStreamInfo GetMediaAttachment(SqliteDataReader reader) + { + var item = new AttachmentStreamInfo + { + Index = reader.GetInt32(1), + Item = null!, + ItemId = reader.GetGuid(0), + }; + + if (reader.TryGetString(2, out var codec)) + { + item.Codec = codec; + } + + if (reader.TryGetString(3, out var codecTag)) + { + item.CodecTag = codecTag; + } + + if (reader.TryGetString(4, out var comment)) + { + item.Comment = comment; + } + + if (reader.TryGetString(5, out var fileName)) + { + item.Filename = fileName; + } + + if (reader.TryGetString(6, out var mimeType)) + { + item.MimeType = mimeType; + } + + return item; + } + private (BaseItemEntity BaseItem, string[] LegacyUserDataKey) GetItem(SqliteDataReader reader) { var entity = new BaseItemEntity() @@ -1214,4 +1332,58 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return image; } + + private class TrackedMigrationStep : IDisposable + { + private readonly string _operationName; + private readonly ILogger _logger; + private readonly Stopwatch _operationTimer; + private bool _disposed; + + public TrackedMigrationStep(string operationName, ILogger logger) + { + _operationName = operationName; + _logger = logger; + _operationTimer = Stopwatch.StartNew(); + logger.LogInformation("Start {OperationName}", operationName); + } + + public bool Disposed + { + get => _disposed; + set => _disposed = value; + } + + public virtual void Dispose() + { + if (Disposed) + { + return; + } + + Disposed = true; + _logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed); + } + } + + private sealed class DatabaseMigrationStep : TrackedMigrationStep + { + public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger) + { + JellyfinDbContext = jellyfinDbContext; + } + + public JellyfinDbContext JellyfinDbContext { get; } + + public override void Dispose() + { + if (Disposed) + { + return; + } + + JellyfinDbContext.Dispose(); + base.Dispose(); + } + } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs new file mode 100644 index 000000000..2d5fc2a0d --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs @@ -0,0 +1,73 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Controller; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for checking if the current instance of Jellyfin is compatiable to be upgraded. +/// </summary> +[JellyfinMigration("2025-04-20T19:30:00", nameof(MigrateLibraryDbCompatibilityCheck))] +public class MigrateLibraryDbCompatibilityCheck : IAsyncMigrationRoutine +{ + private const string DbFilename = "library.db"; + private readonly IStartupLogger _logger; + private readonly IServerApplicationPaths _paths; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateLibraryDbCompatibilityCheck"/> class. + /// </summary> + /// <param name="startupLogger">The startup logger.</param> + /// <param name="paths">The Path service.</param> + public MigrateLibraryDbCompatibilityCheck(IStartupLogger startupLogger, IServerApplicationPaths paths) + { + _logger = startupLogger; + _paths = paths; + } + + /// <inheritdoc/> + public async Task PerformAsync(CancellationToken cancellationToken) + { + var dataPath = _paths.DataPath; + var libraryDbPath = Path.Combine(dataPath, DbFilename); + if (!File.Exists(libraryDbPath)) + { + _logger.LogError("Cannot migrate {LibraryDb} as it does not exist..", libraryDbPath); + return; + } + + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + CheckMigratableVersion(connection); + await connection.CloseAsync().ConfigureAwait(false); + } + + private static void CheckMigratableVersion(SqliteConnection connection) + { + CheckColumnExistance(connection, "TypedBaseItems", "lufs"); + CheckColumnExistance(connection, "TypedBaseItems", "normalizationgain"); + CheckColumnExistance(connection, "mediastreams", "dvversionmajor"); + + static void CheckColumnExistance(SqliteConnection connection, string table, string column) + { + using (var cmd = connection.CreateCommand()) + { +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities + cmd.CommandText = $"Select COUNT(1) FROM pragma_table_xinfo('{table}') WHERE lower(name) = '{column}';"; +#pragma warning restore CA2100 // Review SQL queries for security vulnerabilities + var result = cmd.ExecuteScalar()!; + if (!result.Equals(1L)) + { + throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or above are supported. Please upgrade first to server version 10.10.7 before attempting to upgrade afterwards to 10.11"); + } + } + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 9c2184029..ae93557de 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,94 +1,69 @@ using System; -using System.Globalization; -using System.IO; -using Emby.Server.Implementations.Data; -using MediaBrowser.Controller; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Model.Globalization; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Migrate rating levels to new rating level system. - /// </summary> - internal class MigrateRatingLevels : IMigrationRoutine - { - private const string DbFilename = "library.db"; - private readonly ILogger<MigrateRatingLevels> _logger; - private readonly IServerApplicationPaths _applicationPaths; - private readonly ILocalizationManager _localizationManager; - - public MigrateRatingLevels( - IServerApplicationPaths applicationPaths, - ILoggerFactory loggerFactory, - ILocalizationManager localizationManager) - { - _applicationPaths = applicationPaths; - _localizationManager = localizationManager; - _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{73DAB92A-178B-48CD-B05B-FE18733ACDC8}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "MigrateRatingLevels"; +/// <summary> +/// Migrate rating levels. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T22:00:00", nameof(MigrateRatingLevels))] +[JellyfinMigrationBackup(JellyfinDb = true)] +#pragma warning restore CS0618 // Type or member is obsolete +internal class MigrateRatingLevels : IDatabaseMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly ILocalizationManager _localizationManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public MigrateRatingLevels( + IDbContextFactory<JellyfinDbContext> provider, + IStartupLogger logger, + ILocalizationManager localizationManager) + { + _provider = provider; + _localizationManager = localizationManager; + _logger = logger; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + _logger.LogInformation("Recalculating parental rating levels based on rating string."); + using var context = _provider.CreateDbContext(); + using var transaction = context.Database.BeginTransaction(); + var ratings = context.BaseItems.AsNoTracking().Select(e => e.OfficialRating).Distinct(); + foreach (var rating in ratings) { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) + if (string.IsNullOrEmpty(rating)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) - { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } - } + int? value = null; + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, value)); + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, value)); } - - // Migrate parental rating strings to new levels - _logger.LogInformation("Recalculating parental rating levels based on rating string."); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + else { - var queryResult = connection.Query("SELECT DISTINCT OfficialRating FROM TypedBaseItems"); - foreach (var entry in queryResult) - { - if (!entry.TryGetString(0, out var ratingString) || string.IsNullOrEmpty(ratingString)) - { - connection.Execute("UPDATE TypedBaseItems SET InheritedParentalRatingValue = NULL WHERE OfficialRating IS NULL OR OfficialRating='';"); - } - else - { - var ratingValue = _localizationManager.GetRatingLevel(ratingString)?.ToString(CultureInfo.InvariantCulture) ?? "NULL"; - - using var statement = connection.PrepareStatement("UPDATE TypedBaseItems SET InheritedParentalRatingValue = @Value WHERE OfficialRating = @Rating;"); - statement.TryBind("@Value", ratingValue); - statement.TryBind("@Rating", ratingString); - statement.ExecuteNonQuery(); - } - } - - transaction.Commit(); + var ratingValue = _localizationManager.GetRatingScore(rating); + var score = ratingValue?.Score; + var subScore = ratingValue?.SubScore; + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, score)); + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, subScore)); } } + + transaction.Commit(); } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs index c40560660..e5584fb94 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs @@ -17,206 +17,200 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using JsonSerializer = System.Text.Json.JsonSerializer; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// The migration routine for migrating the user database to EF Core. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T10:00:00", nameof(MigrateUserDb), "5C4B82A2-F053-4009-BD05-B6FCAD82F14C")] +public class MigrateUserDb : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private const string DbFilename = "users.db"; + + private readonly ILogger<MigrateUserDb> _logger; + private readonly IServerApplicationPaths _paths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; + private readonly IXmlSerializer _xmlSerializer; + /// <summary> - /// The migration routine for migrating the user database to EF Core. + /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. /// </summary> - public class MigrateUserDb : IMigrationRoutine + /// <param name="logger">The logger.</param> + /// <param name="paths">The server application paths.</param> + /// <param name="provider">The database provider.</param> + /// <param name="xmlSerializer">The xml serializer.</param> + public MigrateUserDb( + ILogger<MigrateUserDb> logger, + IServerApplicationPaths paths, + IDbContextFactory<JellyfinDbContext> provider, + IXmlSerializer xmlSerializer) { - private const string DbFilename = "users.db"; - - private readonly ILogger<MigrateUserDb> _logger; - private readonly IServerApplicationPaths _paths; - private readonly IDbContextFactory<JellyfinDbContext> _provider; - private readonly IXmlSerializer _xmlSerializer; - - /// <summary> - /// Initializes a new instance of the <see cref="MigrateUserDb"/> class. - /// </summary> - /// <param name="logger">The logger.</param> - /// <param name="paths">The server application paths.</param> - /// <param name="provider">The database provider.</param> - /// <param name="xmlSerializer">The xml serializer.</param> - public MigrateUserDb( - ILogger<MigrateUserDb> logger, - IServerApplicationPaths paths, - IDbContextFactory<JellyfinDbContext> provider, - IXmlSerializer xmlSerializer) - { - _logger = logger; - _paths = paths; - _provider = provider; - _xmlSerializer = xmlSerializer; - } + _logger = logger; + _paths = paths; + _provider = provider; + _xmlSerializer = xmlSerializer; + } - /// <inheritdoc/> - public Guid Id => Guid.Parse("5C4B82A2-F053-4009-BD05-B6FCAD82F14C"); + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); - /// <inheritdoc/> - public string Name => "MigrateUserDatabase"; + using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + { + connection.Open(); + using var dbContext = _provider.CreateDbContext(); - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - _logger.LogInformation("Migrating the user database may take a while, do not stop Jellyfin."); + dbContext.RemoveRange(dbContext.Users); + dbContext.SaveChanges(); - using (var connection = new SqliteConnection($"Filename={Path.Combine(dataPath, DbFilename)}")) + foreach (var entry in queryResult) { - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - var queryResult = connection.Query("SELECT * FROM LocalUsersv2"); + UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); + if (mockup is null) + { + continue; + } - dbContext.RemoveRange(dbContext.Users); - dbContext.SaveChanges(); + var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); + + var configPath = Path.Combine(userDataDir, "config.xml"); + var config = File.Exists(configPath) + ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() + : new UserConfiguration(); + + var policyPath = Path.Combine(userDataDir, "policy.xml"); + var policy = File.Exists(policyPath) + ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() + : new UserPolicy(); + policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( + "Emby.Server.Implementations.Library", + "Jellyfin.Server.Implementations.Users", + StringComparison.Ordinal) + ?? typeof(DefaultAuthenticationProvider).FullName; + + policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; + int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch + { + -1 => null, + 0 => 3, + _ => policy.LoginAttemptsBeforeLockout + }; - foreach (var entry in queryResult) + var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) { - UserMockup? mockup = JsonSerializer.Deserialize<UserMockup>(entry.GetStream(2), JsonDefaults.Options); - if (mockup is null) - { - continue; - } - - var userDataDir = Path.Combine(_paths.UserConfigurationDirectoryPath, mockup.Name); - - var configPath = Path.Combine(userDataDir, "config.xml"); - var config = File.Exists(configPath) - ? (UserConfiguration?)_xmlSerializer.DeserializeFromFile(typeof(UserConfiguration), configPath) ?? new UserConfiguration() - : new UserConfiguration(); - - var policyPath = Path.Combine(userDataDir, "policy.xml"); - var policy = File.Exists(policyPath) - ? (UserPolicy?)_xmlSerializer.DeserializeFromFile(typeof(UserPolicy), policyPath) ?? new UserPolicy() - : new UserPolicy(); - policy.AuthenticationProviderId = policy.AuthenticationProviderId?.Replace( - "Emby.Server.Implementations.Library", - "Jellyfin.Server.Implementations.Users", - StringComparison.Ordinal) - ?? typeof(DefaultAuthenticationProvider).FullName; - - policy.PasswordResetProviderId = typeof(DefaultPasswordResetProvider).FullName; - int? maxLoginAttempts = policy.LoginAttemptsBeforeLockout switch - { - -1 => null, - 0 => 3, - _ => policy.LoginAttemptsBeforeLockout - }; + Id = entry.GetGuid(1), + InternalId = entry.GetInt64(0), + MaxParentalRatingScore = policy.MaxParentalRating, + MaxParentalRatingSubScore = null, + EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, + RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, + InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, + LoginAttemptsBeforeLockout = maxLoginAttempts, + SubtitleMode = config.SubtitleMode, + HidePlayedInLatest = config.HidePlayedInLatest, + EnableLocalPassword = config.EnableLocalPassword, + PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, + DisplayCollectionsView = config.DisplayCollectionsView, + DisplayMissingEpisodes = config.DisplayMissingEpisodes, + AudioLanguagePreference = config.AudioLanguagePreference, + RememberAudioSelections = config.RememberAudioSelections, + EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, + RememberSubtitleSelections = config.RememberSubtitleSelections, + SubtitleLanguagePreference = config.SubtitleLanguagePreference, + Password = mockup.Password, + LastLoginDate = mockup.LastLoginDate, + LastActivityDate = mockup.LastActivityDate + }; + + if (mockup.ImageInfos.Length > 0) + { + ItemImageInfo info = mockup.ImageInfos[0]; - var user = new User(mockup.Name, policy.AuthenticationProviderId!, policy.PasswordResetProviderId!) + user.ProfileImage = new ImageInfo(info.Path) { - Id = entry.GetGuid(1), - InternalId = entry.GetInt64(0), - MaxParentalAgeRating = policy.MaxParentalRating, - EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, - RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, - InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, - LoginAttemptsBeforeLockout = maxLoginAttempts, - SubtitleMode = config.SubtitleMode, - HidePlayedInLatest = config.HidePlayedInLatest, - EnableLocalPassword = config.EnableLocalPassword, - PlayDefaultAudioTrack = config.PlayDefaultAudioTrack, - DisplayCollectionsView = config.DisplayCollectionsView, - DisplayMissingEpisodes = config.DisplayMissingEpisodes, - AudioLanguagePreference = config.AudioLanguagePreference, - RememberAudioSelections = config.RememberAudioSelections, - EnableNextEpisodeAutoPlay = config.EnableNextEpisodeAutoPlay, - RememberSubtitleSelections = config.RememberSubtitleSelections, - SubtitleLanguagePreference = config.SubtitleLanguagePreference, - Password = mockup.Password, - LastLoginDate = mockup.LastLoginDate, - LastActivityDate = mockup.LastActivityDate + LastModified = info.DateModified }; + } - if (mockup.ImageInfos.Length > 0) - { - ItemImageInfo info = mockup.ImageInfos[0]; - - user.ProfileImage = new ImageInfo(info.Path) - { - LastModified = info.DateModified - }; - } - - user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); - user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); - user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); - user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); - user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); - user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); - user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); - user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); - user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); - user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); - user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); - user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); - user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); - user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); - user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); - user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); - user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); - user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); - user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); - user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); - user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); - - foreach (var policyAccessSchedule in policy.AccessSchedules) - { - user.AccessSchedules.Add(policyAccessSchedule); - } - - user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); - user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); - user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); - user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); - user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); - user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); - user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); - user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); - user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); - - dbContext.Users.Add(user); + user.SetPermission(PermissionKind.IsAdministrator, policy.IsAdministrator); + user.SetPermission(PermissionKind.IsHidden, policy.IsHidden); + user.SetPermission(PermissionKind.IsDisabled, policy.IsDisabled); + user.SetPermission(PermissionKind.EnableSharedDeviceControl, policy.EnableSharedDeviceControl); + user.SetPermission(PermissionKind.EnableRemoteAccess, policy.EnableRemoteAccess); + user.SetPermission(PermissionKind.EnableLiveTvManagement, policy.EnableLiveTvManagement); + user.SetPermission(PermissionKind.EnableLiveTvAccess, policy.EnableLiveTvAccess); + user.SetPermission(PermissionKind.EnableMediaPlayback, policy.EnableMediaPlayback); + user.SetPermission(PermissionKind.EnableAudioPlaybackTranscoding, policy.EnableAudioPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableVideoPlaybackTranscoding, policy.EnableVideoPlaybackTranscoding); + user.SetPermission(PermissionKind.EnableContentDeletion, policy.EnableContentDeletion); + user.SetPermission(PermissionKind.EnableContentDownloading, policy.EnableContentDownloading); + user.SetPermission(PermissionKind.EnableSyncTranscoding, policy.EnableSyncTranscoding); + user.SetPermission(PermissionKind.EnableMediaConversion, policy.EnableMediaConversion); + user.SetPermission(PermissionKind.EnableAllChannels, policy.EnableAllChannels); + user.SetPermission(PermissionKind.EnableAllDevices, policy.EnableAllDevices); + user.SetPermission(PermissionKind.EnableAllFolders, policy.EnableAllFolders); + user.SetPermission(PermissionKind.EnableRemoteControlOfOtherUsers, policy.EnableRemoteControlOfOtherUsers); + user.SetPermission(PermissionKind.EnablePlaybackRemuxing, policy.EnablePlaybackRemuxing); + user.SetPermission(PermissionKind.ForceRemoteSourceTranscoding, policy.ForceRemoteSourceTranscoding); + user.SetPermission(PermissionKind.EnablePublicSharing, policy.EnablePublicSharing); + user.SetPermission(PermissionKind.EnableCollectionManagement, policy.EnableCollectionManagement); + + foreach (var policyAccessSchedule in policy.AccessSchedules) + { + user.AccessSchedules.Add(policyAccessSchedule); } - dbContext.SaveChanges(); + user.SetPreference(PreferenceKind.BlockedTags, policy.BlockedTags); + user.SetPreference(PreferenceKind.EnabledChannels, policy.EnabledChannels); + user.SetPreference(PreferenceKind.EnabledDevices, policy.EnabledDevices); + user.SetPreference(PreferenceKind.EnabledFolders, policy.EnabledFolders); + user.SetPreference(PreferenceKind.EnableContentDeletionFromFolders, policy.EnableContentDeletionFromFolders); + user.SetPreference(PreferenceKind.OrderedViews, config.OrderedViews); + user.SetPreference(PreferenceKind.GroupedFolders, config.GroupedFolders); + user.SetPreference(PreferenceKind.MyMediaExcludes, config.MyMediaExcludes); + user.SetPreference(PreferenceKind.LatestItemExcludes, config.LatestItemsExcludes); + + dbContext.Users.Add(user); } - try - { - File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + dbContext.SaveChanges(); + } - var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); - if (File.Exists(journalPath)) - { - File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); - } - } - catch (IOException e) + try + { + File.Move(Path.Combine(dataPath, DbFilename), Path.Combine(dataPath, DbFilename + ".old")); + + var journalPath = Path.Combine(dataPath, DbFilename + "-journal"); + if (File.Exists(journalPath)) { - _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + File.Move(journalPath, Path.Combine(dataPath, DbFilename + ".old-journal")); } } + catch (IOException e) + { + _logger.LogError(e, "Error renaming legacy user database to 'users.db.old'"); + } + } #nullable disable - internal class UserMockup - { - public string Password { get; set; } + internal class UserMockup + { + public string Password { get; set; } - public string EasyPassword { get; set; } + public string EasyPassword { get; set; } - public DateTime? LastLoginDate { get; set; } + public DateTime? LastLoginDate { get; set; } - public DateTime? LastActivityDate { get; set; } + public DateTime? LastActivityDate { get; set; } - public string Name { get; set; } + public string Name { get; set; } - public ItemImageInfo[] ImageInfos { get; set; } - } + public ItemImageInfo[] ImageInfos { get; set; } } } diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs new file mode 100644 index 000000000..6f650f731 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -0,0 +1,295 @@ +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Server.ServerSetupApp; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.IO; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +[JellyfinMigration("2025-04-20T21:00:00", nameof(MoveExtractedFiles))] +public class MoveExtractedFiles : IAsyncMigrationRoutine +{ + private readonly IApplicationPaths _appPaths; + private readonly ILogger _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IPathManager _pathManager; + private readonly IFileSystem _fileSystem; + + /// <summary> + /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class. + /// </summary> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="startupLogger">The startup logger for Startup UI intigration.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + public MoveExtractedFiles( + IApplicationPaths appPaths, + ILogger<MoveExtractedFiles> logger, + IStartupLogger startupLogger, + IPathManager pathManager, + IFileSystem fileSystem, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _appPaths = appPaths; + _logger = startupLogger.With(logger); + _pathManager = pathManager; + _fileSystem = fileSystem; + _dbProvider = dbProvider; + } + + private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles"); + + private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); + + /// <inheritdoc /> + public async Task PerformAsync(CancellationToken cancellationToken) + { + const int Limit = 5000; + int itemCount = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder); + _logger.LogInformation("Checking {Count} items for movable extracted files.", records); + + // Make sure directories exist + Directory.CreateDirectory(SubtitleCachePath); + Directory.CreateDirectory(AttachmentCachePath); + + await foreach (var result in context.BaseItems + .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal)) + .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder) + .Select(b => new + { + b.Id, + b.Path, + b.MediaStreams + }) + .OrderBy(e => e.Id) + .WithPartitionProgress((partition) => _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", partition * Limit, itemCount, sw.Elapsed)) + .PartitionEagerAsync(Limit, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + if (MoveSubtitleAndAttachmentFiles(result.Id, result.Path, result.MediaStreams, context)) + { + itemCount++; + } + } + + _logger.LogInformation("Moved files for {Count} items in {Time}", itemCount, sw.Elapsed); + + // Get all subdirectories with 1 character names (those are the legacy directories) + var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList(); + subdirectories.AddRange(Directory.GetDirectories(AttachmentCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == AttachmentCachePath.Length + 2)); + + // Remove all legacy subdirectories + foreach (var subdir in subdirectories) + { + Directory.Delete(subdir, true); + } + + // Remove old cache path + var attachmentCachePath = Path.Join(_appPaths.CachePath, "attachments"); + if (Directory.Exists(attachmentCachePath)) + { + Directory.Delete(attachmentCachePath, true); + } + + _logger.LogInformation("Cleaned up left over subtitles and attachments."); + } + + private bool MoveSubtitleAndAttachmentFiles(Guid id, string? path, ICollection<MediaStreamInfo>? mediaStreams, JellyfinDbContext context) + { + var itemIdString = id.ToString("N", CultureInfo.InvariantCulture); + var modified = false; + if (mediaStreams is not null) + { + foreach (var mediaStream in mediaStreams) + { + if (mediaStream.Codec is null) + { + continue; + } + + var mediaStreamIndex = mediaStream.StreamIndex; + var extension = GetSubtitleExtension(mediaStream.Codec); + var oldSubtitleCachePath = GetOldSubtitleCachePath(path, mediaStreamIndex, extension); + if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath)) + { + continue; + } + + var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension); + if (File.Exists(newSubtitleCachePath)) + { + File.Delete(oldSubtitleCachePath); + } + else + { + var newDirectory = Path.GetDirectoryName(newSubtitleCachePath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldSubtitleCachePath, newSubtitleCachePath, false); + _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, id, oldSubtitleCachePath, newSubtitleCachePath); + + modified = true; + } + } + } + } + +#pragma warning disable CA1309 // Use ordinal string comparison + var attachments = context.AttachmentStreamInfos.Where(a => a.ItemId.Equals(id) && !string.Equals(a.Codec, "mjpeg")).ToList(); +#pragma warning restore CA1309 // Use ordinal string comparison + var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.Filename) + && (a.Filename.Contains('/', StringComparison.OrdinalIgnoreCase) || a.Filename.Contains('\\', StringComparison.OrdinalIgnoreCase))); + foreach (var attachment in attachments) + { + var attachmentIndex = attachment.Index; + var oldAttachmentPath = GetOldAttachmentDataPath(path, attachmentIndex); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + continue; + } + } + + var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.Filename ?? attachmentIndex.ToString(CultureInfo.InvariantCulture)); + if (File.Exists(newAttachmentPath)) + { + File.Delete(oldAttachmentPath); + } + else + { + var newDirectory = Path.GetDirectoryName(newAttachmentPath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldAttachmentPath, newAttachmentPath, false); + _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, id, oldAttachmentPath, newAttachmentPath); + + modified = true; + } + } + } + + return modified; + } + + private string? GetOldAttachmentDataPath(string? mediaPath, int attachmentStreamIndex) + { + if (mediaPath is null) + { + return null; + } + + string filename; + if (_fileSystem.IsPathFile(mediaPath)) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(mediaPath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping attachment at index {Index} for {Path}: {Exception}", attachmentStreamIndex, mediaPath, e.Message); + + return null; + } + + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + else + { + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + + return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename); + } + + private string? GetOldAttachmentCachePath(string mediaSourceId, AttachmentStreamInfo attachment, bool shouldExtractOneByOne) + { + var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId); + if (shouldExtractOneByOne) + { + return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture)); + } + + if (string.IsNullOrEmpty(attachment.Filename)) + { + return null; + } + + return Path.Join(attachmentFolderPath, attachment.Filename); + } + + private string? GetOldSubtitleCachePath(string? path, int streamIndex, string outputSubtitleExtension) + { + if (path is null) + { + return null; + } + + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(path); + } + catch (IOException e) + { + _logger.LogDebug("Skipping subtitle at index {Index} for {Path}: {Exception}", streamIndex, path, e.Message); + + return null; + } + + var ticksParam = string.Empty; + ReadOnlySpan<char> filename = new Guid(MD5.HashData(Encoding.Unicode.GetBytes(path + "_" + streamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture) + ticksParam))) + outputSubtitleExtension; + + return Path.Join(SubtitleCachePath, filename[..1], filename); + } + + private static string GetSubtitleExtension(string codec) + { + if (codec.ToLower(CultureInfo.InvariantCulture).Equals("ass", StringComparison.OrdinalIgnoreCase) + || codec.ToLower(CultureInfo.InvariantCulture).Equals("ssa", StringComparison.OrdinalIgnoreCase)) + { + return "." + codec; + } + else if (codec.Contains("pgs", StringComparison.OrdinalIgnoreCase)) + { + return ".sup"; + } + else + { + return ".srt"; + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index f4ebac377..a674aa928 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,7 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; -using MediaBrowser.Common; +using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -16,12 +16,15 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to move trickplay files to the new directory. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T23:00:00", nameof(MoveTrickplayFiles), RunMigrationOnSetup = true)] public class MoveTrickplayFiles : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; private readonly ILibraryManager _libraryManager; - private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IStartupLogger _logger; /// <summary> /// Initializes a new instance of the <see cref="MoveTrickplayFiles"/> class. @@ -30,7 +33,11 @@ public class MoveTrickplayFiles : IMigrationRoutine /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> /// <param name="logger">The logger.</param> - public MoveTrickplayFiles(ITrickplayManager trickplayManager, IFileSystem fileSystem, ILibraryManager libraryManager, ILogger<MoveTrickplayFiles> logger) + public MoveTrickplayFiles( + ITrickplayManager trickplayManager, + IFileSystem fileSystem, + ILibraryManager libraryManager, + IStartupLogger logger) { _trickplayManager = trickplayManager; _fileSystem = fileSystem; @@ -39,18 +46,9 @@ public class MoveTrickplayFiles : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("9540D44A-D8DC-11EF-9CBB-B77274F77C52"); - - /// <inheritdoc /> - public string Name => "MoveTrickplayFiles"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { - const int Limit = 100; + const int Limit = 5000; int itemCount = 0, offset = 0, previousCount; var sw = Stopwatch.StartNew(); @@ -65,9 +63,6 @@ public class MoveTrickplayFiles : IMigrationRoutine do { var trickplayInfos = _trickplayManager.GetTrickplayItemsAsync(Limit, offset).GetAwaiter().GetResult(); - previousCount = trickplayInfos.Count; - offset += Limit; - trickplayQuery.ItemIds = trickplayInfos.Select(i => i.ItemId).Distinct().ToArray(); var items = _libraryManager.GetItemList(trickplayQuery); foreach (var trickplayInfo in trickplayInfos) @@ -78,24 +73,32 @@ public class MoveTrickplayFiles : IMigrationRoutine continue; } - if (++itemCount % 1_000 == 0) - { - _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); - } - + var moved = false; var oldPath = GetOldTrickplayDirectory(item, trickplayInfo.Width); var newPath = _trickplayManager.GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); if (_fileSystem.DirectoryExists(oldPath)) { _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; } oldPath = GetNewOldTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false); if (_fileSystem.DirectoryExists(oldPath)) { _fileSystem.MoveDirectory(oldPath, newPath); + moved = true; + } + + if (moved) + { + itemCount++; } } + + offset += Limit; + previousCount = trickplayInfos.Count; + + _logger.LogInformation("Checked: {Checked} - Moved: {Count} - Time: {Time}", offset, itemCount, sw.Elapsed); } while (previousCount == Limit); _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed); diff --git a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs index 9cfaec46f..ebf4a2780 100644 --- a/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/ReaddDefaultPluginRepository.cs @@ -2,48 +2,41 @@ using System; using MediaBrowser.Controller.Configuration; using MediaBrowser.Model.Updates; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to initialize system configuration with the default plugin repository. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T11:00:00", nameof(ReaddDefaultPluginRepository), "5F86E7F6-D966-4C77-849D-7A7B40B68C4E", RunMigrationOnSetup = true)] +public class ReaddDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { + private readonly IServerConfigurationManager _serverConfigurationManager; + + private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo + { + Name = "Jellyfin Stable", + Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" + }; + /// <summary> - /// Migration to initialize system configuration with the default plugin repository. + /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. /// </summary> - public class ReaddDefaultPluginRepository : IMigrationRoutine + /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) { - private readonly IServerConfigurationManager _serverConfigurationManager; - - private readonly RepositoryInfo _defaultRepositoryInfo = new RepositoryInfo - { - Name = "Jellyfin Stable", - Url = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json" - }; - - /// <summary> - /// Initializes a new instance of the <see cref="ReaddDefaultPluginRepository"/> class. - /// </summary> - /// <param name="serverConfigurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> - public ReaddDefaultPluginRepository(IServerConfigurationManager serverConfigurationManager) - { - _serverConfigurationManager = serverConfigurationManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("5F86E7F6-D966-4C77-849D-7A7B40B68C4E"); - - /// <inheritdoc/> - public string Name => "ReaddDefaultPluginRepository"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => true; + _serverConfigurationManager = serverConfigurationManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + // Only add if repository list is empty + if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) { - // Only add if repository list is empty - if (_serverConfigurationManager.Configuration.PluginRepositories.Length == 0) - { - _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; - _serverConfigurationManager.SaveConfiguration(); - } + _serverConfigurationManager.Configuration.PluginRepositories = new[] { _defaultRepositoryInfo }; + _serverConfigurationManager.SaveConfiguration(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs new file mode 100644 index 000000000..b23a7dbc4 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/RefreshInternalDateModified.cs @@ -0,0 +1,131 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to re-read creation dates for library items with internal metadata paths. +/// </summary> +[JellyfinMigration("2025-04-20T23:00:00", nameof(RefreshInternalDateModified))] +public class RefreshInternalDateModified : IDatabaseMigrationRoutine +{ + private readonly ILogger<RefreshInternalDateModified> _logger; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IFileSystem _fileSystem; + private readonly IServerApplicationHost _applicationHost; + private readonly bool _useFileCreationTimeForDateAdded; + + private IReadOnlyList<string> _internalTypes = [ + typeof(Genre).FullName!, + typeof(MusicGenre).FullName!, + typeof(MusicArtist).FullName!, + typeof(People).FullName!, + typeof(Studio).FullName! + ]; + + private IReadOnlyList<string> _internalPaths; + + /// <summary> + /// Initializes a new instance of the <see cref="RefreshInternalDateModified"/> class. + /// </summary> + /// <param name="applicationHost">Instance of the <see cref="IServerApplicationHost"/> interface.</param> + /// <param name="applicationPaths">Instance of the <see cref="IServerApplicationPaths"/> interface.</param> + /// <param name="configurationManager">Instance of the <see cref="IServerConfigurationManager"/> interface.</param> + /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param> + public RefreshInternalDateModified( + IServerApplicationHost applicationHost, + IServerApplicationPaths applicationPaths, + IServerConfigurationManager configurationManager, + IDbContextFactory<JellyfinDbContext> dbProvider, + ILogger<RefreshInternalDateModified> logger, + IFileSystem fileSystem) + { + _dbProvider = dbProvider; + _logger = logger; + _fileSystem = fileSystem; + _applicationHost = applicationHost; + _internalPaths = [ + applicationPaths.ArtistsPath, + applicationPaths.GenrePath, + applicationPaths.MusicGenrePath, + applicationPaths.StudioPath, + applicationPaths.PeoplePath + ]; + _useFileCreationTimeForDateAdded = configurationManager.GetMetadataConfiguration().UseFileCreationTimeForDateAdded; + } + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 5000; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => _internalTypes.Contains(b.Type)); + _logger.LogInformation("Checking if {Count} potentially internal items require refreshed DateModified", records); + + do + { + var results = context.BaseItems + .Where(b => _internalTypes.Contains(b.Type)) + .OrderBy(e => e.Id) + .Skip(offset) + .Take(Limit) + .ToList(); + + foreach (var item in results) + { + var itemPath = item.Path; + if (itemPath is not null) + { + var realPath = _applicationHost.ExpandVirtualPath(item.Path); + if (_internalPaths.Any(path => realPath.StartsWith(path, StringComparison.Ordinal))) + { + var writeTime = _fileSystem.GetLastWriteTimeUtc(realPath); + var itemModificationTime = item.DateModified; + if (writeTime != itemModificationTime) + { + _logger.LogDebug("Reset file modification date: Old: {Old} - New: {New} - Path: {Path}", itemModificationTime, writeTime, realPath); + item.DateModified = writeTime; + if (_useFileCreationTimeForDateAdded) + { + item.DateCreated = _fileSystem.GetCreationTimeUtc(realPath); + } + + itemCount++; + } + } + } + } + + offset += Limit; + if (offset > records) + { + offset = records; + } + + _logger.LogInformation("Checked: {Count} - Refreshed: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed); + } while (offset < records); + + context.SaveChanges(); + + _logger.LogInformation("Refreshed DateModified for {Count} items in {Time}", itemCount, sw.Elapsed); + } +} diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs index 52fb93d59..b626c473e 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDownloadImagesInAdvance.cs @@ -3,50 +3,43 @@ using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines -{ - /// <summary> - /// Removes the old 'RemoveDownloadImagesInAdvance' from library options. - /// </summary> - internal class RemoveDownloadImagesInAdvance : IMigrationRoutine - { - private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; - private readonly ILibraryManager _libraryManager; - - public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) - { - _logger = logger; - _libraryManager = libraryManager; - } - - /// <inheritdoc/> - public Guid Id => Guid.Parse("{A81F75E0-8F43-416F-A5E8-516CCAB4D8CC}"); +namespace Jellyfin.Server.Migrations.Routines; - /// <inheritdoc/> - public string Name => "RemoveDownloadImagesInAdvance"; +/// <summary> +/// Removes the old 'RemoveDownloadImagesInAdvance' from library options. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T13:00:00", nameof(RemoveDownloadImagesInAdvance), "A81F75E0-8F43-416F-A5E8-516CCAB4D8CC")] +internal class RemoveDownloadImagesInAdvance : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete +{ + private readonly ILogger<RemoveDownloadImagesInAdvance> _logger; + private readonly ILibraryManager _libraryManager; - /// <inheritdoc/> - public bool PerformOnNewInstall => false; + public RemoveDownloadImagesInAdvance(ILogger<RemoveDownloadImagesInAdvance> logger, ILibraryManager libraryManager) + { + _logger = logger; + _libraryManager = libraryManager; + } - /// <inheritdoc/> - public void Perform() + /// <inheritdoc/> + public void Perform() + { + var virtualFolders = _libraryManager.GetVirtualFolders(false); + _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); + foreach (var virtualFolder in virtualFolders) { - var virtualFolders = _libraryManager.GetVirtualFolders(false); - _logger.LogInformation("Removing 'RemoveDownloadImagesInAdvance' settings in all the libraries"); - foreach (var virtualFolder in virtualFolders) + // Some virtual folders don't have a proper item id. + if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) { - // Some virtual folders don't have a proper item id. - if (!Guid.TryParse(virtualFolder.ItemId, out var folderId)) - { - continue; - } - - var libraryOptions = virtualFolder.LibraryOptions; - var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); - // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. - collectionFolder.UpdateLibraryOptions(libraryOptions); - _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); + continue; } + + var libraryOptions = virtualFolder.LibraryOptions; + var collectionFolder = _libraryManager.GetItemById<CollectionFolder>(folderId) ?? throw new InvalidOperationException("Failed to find CollectionFolder"); + // The property no longer exists in LibraryOptions, so we just re-save the options to get old data removed. + collectionFolder.UpdateLibraryOptions(libraryOptions); + _logger.LogInformation("Removed from '{VirtualFolder}'", virtualFolder.Name); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs index 7b0d9456d..c9e66d0cf 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicateExtras.cs @@ -7,77 +7,70 @@ using MediaBrowser.Controller; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; -namespace Jellyfin.Server.Migrations.Routines +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. +/// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T08:00:00", nameof(RemoveDuplicateExtras), "ACBE17B7-8435-4A83-8B64-6FCF162CB9BD")] +internal class RemoveDuplicateExtras : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - /// <summary> - /// Remove duplicate entries which were caused by a bug where a file was considered to be an "Extra" to itself. - /// </summary> - internal class RemoveDuplicateExtras : IMigrationRoutine + private const string DbFilename = "library.db"; + private readonly ILogger<RemoveDuplicateExtras> _logger; + private readonly IServerApplicationPaths _paths; + + public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) { - private const string DbFilename = "library.db"; - private readonly ILogger<RemoveDuplicateExtras> _logger; - private readonly IServerApplicationPaths _paths; + _logger = logger; + _paths = paths; + } - public RemoveDuplicateExtras(ILogger<RemoveDuplicateExtras> logger, IServerApplicationPaths paths) + /// <inheritdoc/> + public void Perform() + { + var dataPath = _paths.DataPath; + var dbPath = Path.Combine(dataPath, DbFilename); + using var connection = new SqliteConnection($"Filename={dbPath}"); + connection.Open(); + using (var transaction = connection.BeginTransaction()) { - _logger = logger; - _paths = paths; - } + // Query the database for the ids of duplicate extras + var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); + var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - /// <inheritdoc/> - public Guid Id => Guid.Parse("{ACBE17B7-8435-4A83-8B64-6FCF162CB9BD}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicateExtras"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> - public void Perform() - { - var dataPath = _paths.DataPath; - var dbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) + // Do nothing if no duplicate extras were detected + if (bads.Length == 0) { - // Query the database for the ids of duplicate extras - var queryResult = connection.Query("SELECT t1.Path FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video'"); - var bads = string.Join(", ", queryResult.Select(x => x.GetString(0))); - - // Do nothing if no duplicate extras were detected - if (bads.Length == 0) - { - _logger.LogInformation("No duplicate extras detected, skipping migration."); - return; - } + _logger.LogInformation("No duplicate extras detected, skipping migration."); + return; + } - // Back up the database before deleting any entries - for (int i = 1; ; i++) + // Back up the database before deleting any entries + for (int i = 1; ; i++) + { + var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); + if (!File.Exists(bakPath)) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + try { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } + File.Copy(dbPath, bakPath); + _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); + throw; } } - - // Delete all duplicate extras - _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); - connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); - transaction.Commit(); } + + // Delete all duplicate extras + _logger.LogInformation("Removing found duplicated extras for the following items: {DuplicateExtras}", bads); + connection.Execute("DELETE FROM TypedBaseItems WHERE rowid IN (SELECT t1.rowid FROM TypedBaseItems AS t1, TypedBaseItems AS t2 WHERE t1.Path=t2.Path AND t1.Type!=t2.Type AND t1.Type='MediaBrowser.Controller.Entities.Video')"); + transaction.Commit(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs index f84bccc25..23f212424 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs @@ -1,44 +1,33 @@ using System; using System.Linq; using System.Threading; - using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Playlists; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Remove duplicate playlist entries. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T19:00:00", nameof(RemoveDuplicatePlaylistChildren), "96C156A2-7A13-4B3B-A8B8-FB80C94D20C0")] internal class RemoveDuplicatePlaylistChildren : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { - private readonly ILogger<RemoveDuplicatePlaylistChildren> _logger; private readonly ILibraryManager _libraryManager; private readonly IPlaylistManager _playlistManager; public RemoveDuplicatePlaylistChildren( - ILogger<RemoveDuplicatePlaylistChildren> logger, ILibraryManager libraryManager, IPlaylistManager playlistManager) { - _logger = logger; _libraryManager = libraryManager; _playlistManager = playlistManager; } /// <inheritdoc/> - public Guid Id => Guid.Parse("{96C156A2-7A13-4B3B-A8B8-FB80C94D20C0}"); - - /// <inheritdoc/> - public string Name => "RemoveDuplicatePlaylistChildren"; - - /// <inheritdoc/> - public bool PerformOnNewInstall => false; - - /// <inheritdoc/> public void Perform() { var playlists = _libraryManager.GetItemList(new InternalItemsQuery diff --git a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs index 7e8c8ac87..f58cf2741 100644 --- a/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs +++ b/Jellyfin.Server/Migrations/Routines/UpdateDefaultPluginRepository.cs @@ -6,7 +6,10 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to update the default Jellyfin plugin repository. /// </summary> +#pragma warning disable CS0618 // Type or member is obsolete +[JellyfinMigration("2025-04-20T17:00:00", nameof(UpdateDefaultPluginRepository), "852816E0-2712-49A9-9240-C6FC5FCAD1A8", RunMigrationOnSetup = true)] public class UpdateDefaultPluginRepository : IMigrationRoutine +#pragma warning restore CS0618 // Type or member is obsolete { private const string NewRepositoryUrl = "https://repo.jellyfin.org/files/plugin/manifest.json"; private const string OldRepositoryUrl = "https://repo.jellyfin.org/releases/plugin/manifest-stable.json"; @@ -23,15 +26,6 @@ public class UpdateDefaultPluginRepository : IMigrationRoutine } /// <inheritdoc /> - public Guid Id => new("852816E0-2712-49A9-9240-C6FC5FCAD1A8"); - - /// <inheritdoc /> - public string Name => "UpdateDefaultPluginRepository10.9"; - - /// <inheritdoc /> - public bool PerformOnNewInstall => true; - - /// <inheritdoc /> public void Perform() { var updated = false; |