diff options
Diffstat (limited to 'Jellyfin.Server')
10 files changed, 951 insertions, 260 deletions
diff --git a/Jellyfin.Server/Filters/AdditionalModelFilter.cs b/Jellyfin.Server/Filters/AdditionalModelFilter.cs index bf38f741c..4cd0fc231 100644 --- a/Jellyfin.Server/Filters/AdditionalModelFilter.cs +++ b/Jellyfin.Server/Filters/AdditionalModelFilter.cs @@ -25,7 +25,7 @@ namespace Jellyfin.Server.Filters public class AdditionalModelFilter : IDocumentFilter { // Array of options that should not be visible in the api spec. - private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions) }; + private static readonly Type[] _ignoredConfigurations = { typeof(MigrationOptions), typeof(MediaBrowser.Model.Branding.BrandingOptions) }; private readonly IServerConfigurationManager _serverConfigurationManager; /// <summary> diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs index fd540c9c0..c3a2e1bc4 100644 --- a/Jellyfin.Server/Migrations/MigrationRunner.cs +++ b/Jellyfin.Server/Migrations/MigrationRunner.cs @@ -29,7 +29,8 @@ namespace Jellyfin.Server.Migrations typeof(PreStartupRoutines.CreateNetworkConfiguration), typeof(PreStartupRoutines.MigrateMusicBrainzTimeout), typeof(PreStartupRoutines.MigrateNetworkConfiguration), - typeof(PreStartupRoutines.MigrateEncodingOptions) + typeof(PreStartupRoutines.MigrateEncodingOptions), + typeof(PreStartupRoutines.RenameEnableGroupingIntoCollections) }; /// <summary> @@ -48,13 +49,15 @@ namespace Jellyfin.Server.Migrations typeof(Routines.RemoveDownloadImagesInAdvance), typeof(Routines.MigrateAuthenticationDb), typeof(Routines.FixPlaylistOwner), - typeof(Routines.MigrateRatingLevels), typeof(Routines.AddDefaultCastReceivers), typeof(Routines.UpdateDefaultPluginRepository), typeof(Routines.FixAudioData), - typeof(Routines.MoveTrickplayFiles), typeof(Routines.RemoveDuplicatePlaylistChildren), typeof(Routines.MigrateLibraryDb), + typeof(Routines.MoveExtractedFiles), + typeof(Routines.MigrateRatingLevels), + typeof(Routines.MoveTrickplayFiles), + typeof(Routines.MigrateKeyframeData), }; /// <summary> diff --git a/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs new file mode 100644 index 000000000..0a37b35a6 --- /dev/null +++ b/Jellyfin.Server/Migrations/PreStartupRoutines/RenameEnableGroupingIntoCollections.cs @@ -0,0 +1,63 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; +using Emby.Server.Implementations; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.PreStartupRoutines; + +/// <inheritdoc /> +public class RenameEnableGroupingIntoCollections : IMigrationRoutine +{ + private readonly ServerApplicationPaths _applicationPaths; + private readonly ILogger<RenameEnableGroupingIntoCollections> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="RenameEnableGroupingIntoCollections"/> class. + /// </summary> + /// <param name="applicationPaths">An instance of <see cref="ServerApplicationPaths"/>.</param> + /// <param name="loggerFactory">An instance of the <see cref="ILoggerFactory"/> interface.</param> + public RenameEnableGroupingIntoCollections(ServerApplicationPaths applicationPaths, ILoggerFactory loggerFactory) + { + _applicationPaths = applicationPaths; + _logger = loggerFactory.CreateLogger<RenameEnableGroupingIntoCollections>(); + } + + /// <inheritdoc /> + public Guid Id => Guid.Parse("E73B777D-CD5C-4E71-957A-B86B3660B7CF"); + + /// <inheritdoc /> + public string Name => nameof(RenameEnableGroupingIntoCollections); + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + string path = Path.Combine(_applicationPaths.ConfigurationDirectoryPath, "system.xml"); + if (!File.Exists(path)) + { + _logger.LogWarning("Configuration file not found: {Path}", path); + return; + } + + try + { + XDocument xmlDocument = XDocument.Load(path); + var element = xmlDocument.Descendants("EnableGroupingIntoCollections").FirstOrDefault(); + if (element is not null) + { + element.Name = "EnableGroupingMoviesIntoCollections"; + _logger.LogInformation("The tag <EnableGroupingIntoCollections> was successfully renamed to <EnableGroupingMoviesIntoCollections>."); + xmlDocument.Save(path); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "An error occurred while updating the XML file: {Message}", ex.Message); + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs new file mode 100644 index 000000000..b8e69db8e --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using Jellyfin.Data.Enums; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions.Json; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +public class MigrateKeyframeData : IDatabaseMigrationRoutine +{ + private readonly ILibraryManager _libraryManager; + private readonly ILogger<MoveTrickplayFiles> _logger; + private readonly IApplicationPaths _appPaths; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + + /// <summary> + /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class. + /// </summary> + /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="dbProvider">The EFCore db factory.</param> + public MigrateKeyframeData( + ILibraryManager libraryManager, + ILogger<MoveTrickplayFiles> logger, + IApplicationPaths appPaths, + IDbContextFactory<JellyfinDbContext> dbProvider) + { + _libraryManager = libraryManager; + _logger = logger; + _appPaths = appPaths; + _dbProvider = dbProvider; + } + + private string KeyframeCachePath => Path.Combine(_appPaths.DataPath, "keyframes"); + + /// <inheritdoc /> + public Guid Id => new("EA4bCAE1-09A4-428E-9B90-4B4FD2EA1B24"); + + /// <inheritdoc /> + public string Name => "MigrateKeyframeData"; + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 100; + int itemCount = 0, offset = 0, previousCount; + + var sw = Stopwatch.StartNew(); + var itemsQuery = new InternalItemsQuery + { + MediaTypes = [MediaType.Video], + SourceTypes = [SourceType.Library], + IsVirtualItem = false, + IsFolder = false + }; + + using var context = _dbProvider.CreateDbContext(); + context.KeyframeData.ExecuteDelete(); + using var transaction = context.Database.BeginTransaction(); + List<KeyframeData> keyframes = []; + + do + { + var result = _libraryManager.GetItemsResult(itemsQuery); + _logger.LogInformation("Importing keyframes for {Count} items", result.TotalRecordCount); + + var items = result.Items; + previousCount = items.Count; + offset += Limit; + foreach (var item in items) + { + if (TryGetKeyframeData(item, out var data)) + { + keyframes.Add(data); + } + + if (++itemCount % 10_000 == 0) + { + context.KeyframeData.AddRange(keyframes); + keyframes.Clear(); + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + } + } + } while (previousCount == Limit); + + context.KeyframeData.AddRange(keyframes); + context.SaveChanges(); + transaction.Commit(); + + _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed); + + if (Directory.Exists(KeyframeCachePath)) + { + Directory.Delete(KeyframeCachePath, true); + } + } + + private bool TryGetKeyframeData(BaseItem item, [NotNullWhen(true)] out KeyframeData? data) + { + data = null; + var path = item.Path; + if (!string.IsNullOrEmpty(path)) + { + var cachePath = GetCachePath(KeyframeCachePath, path); + if (TryReadFromCache(cachePath, out var keyframeData)) + { + data = new() + { + ItemId = item.Id, + KeyframeTicks = keyframeData.KeyframeTicks.ToList(), + TotalDuration = keyframeData.TotalDuration + }; + + return true; + } + } + + return false; + } + + private string? GetCachePath(string keyframeCachePath, string filePath) + { + DateTime? lastWriteTimeUtc; + try + { + lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping {Path}: {Exception}", filePath, e.Message); + + return null; + } + + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string? cachePath, [NotNullWhen(true)] out MediaEncoding.Keyframes.KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<MediaEncoding.Keyframes.KeyframeData>(bytes, _jsonOptions); + + return cachedResult is not null; + } + + cachedResult = null; + + return false; + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs index 427f04f9d..3fc9bea84 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs @@ -73,273 +73,352 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine var dataPath = _paths.DataPath; var libraryDbPath = Path.Combine(dataPath, DbFilename); - using var connection = new SqliteConnection($"Filename={libraryDbPath}"); - var migrationTotalTime = TimeSpan.Zero; + using var connection = new SqliteConnection($"Filename={libraryDbPath};Mode=ReadOnly"); - var stopwatch = new Stopwatch(); - stopwatch.Start(); + var fullOperationTimer = new Stopwatch(); + fullOperationTimer.Start(); - connection.Open(); - using var dbContext = _provider.CreateDbContext(); - - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving UserData entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving TypedBaseItem."); - const string typedBaseItemsQuery = """ - SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, - IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, - PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, - ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, - Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, - DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, - PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, - ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems - """; - dbContext.BaseItems.ExecuteDelete(); + using (var operation = GetPreparedDbContext("Cleanup database")) + { + operation.JellyfinDbContext.AttachmentStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.BaseItems.ExecuteDelete(); + operation.JellyfinDbContext.ItemValues.ExecuteDelete(); + operation.JellyfinDbContext.UserData.ExecuteDelete(); + operation.JellyfinDbContext.MediaStreamInfos.ExecuteDelete(); + operation.JellyfinDbContext.Peoples.ExecuteDelete(); + operation.JellyfinDbContext.PeopleBaseItemMap.ExecuteDelete(); + operation.JellyfinDbContext.Chapters.ExecuteDelete(); + operation.JellyfinDbContext.AncestorIds.ExecuteDelete(); + } var legacyBaseItemWithUserKeys = new Dictionary<string, BaseItemEntity>(); - foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) - { - var baseItem = GetItem(dto); - dbContext.BaseItems.Add(baseItem.BaseItem); - foreach (var dataKey in baseItem.LegacyUserDataKey) + connection.Open(); + + var baseItemIds = new HashSet<Guid>(); + using (var operation = GetPreparedDbContext("moving TypedBaseItem")) + { + const string typedBaseItemsQuery = + """ + SELECT guid, type, data, StartDate, EndDate, ChannelId, IsMovie, + IsSeries, EpisodeTitle, IsRepeat, CommunityRating, CustomRating, IndexNumber, IsLocked, PreferredMetadataLanguage, + PreferredMetadataCountryCode, Width, Height, DateLastRefreshed, Name, Path, PremiereDate, Overview, ParentIndexNumber, + ProductionYear, OfficialRating, ForcedSortName, RunTimeTicks, Size, DateCreated, DateModified, Genres, ParentId, TopParentId, + Audio, ExternalServiceId, IsInMixedFolder, DateLastSaved, LockedFields, Studios, Tags, TrailerTypes, OriginalTitle, PrimaryVersionId, + DateLastMediaAdded, Album, LUFS, NormalizationGain, CriticRating, IsVirtualItem, SeriesName, UserDataKey, SeasonName, SeasonId, SeriesId, + PresentationUniqueKey, InheritedParentalRatingValue, ExternalSeriesId, Tagline, ProviderIds, Images, ProductionLocations, ExtraIds, TotalBitrate, + ExtraType, Artists, AlbumArtists, ExternalId, SeriesPresentationUniqueKey, ShowId, OwnerId, MediaType, SortName, CleanName, UnratedType FROM TypedBaseItems + """; + using (new TrackedMigrationStep("Loading TypedBaseItems", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(typedBaseItemsQuery)) + { + var baseItem = GetItem(dto); + operation.JellyfinDbContext.BaseItems.Add(baseItem.BaseItem); + baseItemIds.Add(baseItem.BaseItem.Id); + foreach (var dataKey in baseItem.LegacyUserDataKey) + { + legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + } + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.BaseItems.Local.Count} BaseItem entries", _logger)) { - legacyBaseItemWithUserKeys[dataKey] = baseItem.BaseItem; + operation.JellyfinDbContext.SaveChanges(); } } - _logger.LogInformation("Try saving {0} BaseItem entries.", dbContext.BaseItems.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving BaseItems entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (var operation = GetPreparedDbContext("moving ItemValues")) + { + // do not migrate inherited types as they are now properly mapped in search and lookup. + const string itemValueQuery = + """ + SELECT ItemId, Type, Value, CleanValue FROM ItemValues + WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) + """; - _logger.LogInformation("Start moving ItemValues."); - // do not migrate inherited types as they are now properly mapped in search and lookup. - const string itemValueQuery = - """ - SELECT ItemId, Type, Value, CleanValue FROM ItemValues - WHERE Type <> 6 AND EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = ItemValues.ItemId) - """; - dbContext.ItemValues.ExecuteDelete(); + // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. + var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + using (new TrackedMigrationStep("loading ItemValues", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) + { + var itemId = dto.GetGuid(0); + var entity = GetItemValue(dto); + var key = ((int)entity.Type, entity.CleanValue); + if (!localItems.TryGetValue(key, out var existing)) + { + localItems[key] = existing = (entity, []); + } - // EFCores local lookup sucks. We cannot use context.ItemValues.Local here because its just super slow. - var localItems = new Dictionary<(int Type, string CleanValue), (Database.Implementations.Entities.ItemValue ItemValue, List<Guid> ItemIds)>(); + existing.ItemIds.Add(itemId); + } - foreach (SqliteDataReader dto in connection.Query(itemValueQuery)) - { - var itemId = dto.GetGuid(0); - var entity = GetItemValue(dto); - var key = ((int)entity.Type, entity.CleanValue); - if (!localItems.TryGetValue(key, out var existing)) - { - localItems[key] = existing = (entity, []); + foreach (var item in localItems) + { + operation.JellyfinDbContext.ItemValues.Add(item.Value.ItemValue); + operation.JellyfinDbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + { + Item = null!, + ItemValue = null!, + ItemId = f, + ItemValueId = item.Value.ItemValue.ItemValueId + })); + } } - existing.ItemIds.Add(itemId); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.ItemValues.Local.Count} ItemValues entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - foreach (var item in localItems) + using (var operation = GetPreparedDbContext("moving UserData")) { - dbContext.ItemValues.Add(item.Value.ItemValue); - dbContext.ItemValuesMap.AddRange(item.Value.ItemIds.Distinct().Select(f => new ItemValueMap() + var queryResult = connection.Query( + """ + SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) + """); + + using (new TrackedMigrationStep("loading UserData", _logger)) { - Item = null!, - ItemValue = null!, - ItemId = f, - ItemValueId = item.Value.ItemValue.ItemValueId - })); - } + var users = operation.JellyfinDbContext.Users.AsNoTracking().ToImmutableArray(); + var userIdBlacklist = new HashSet<int>(); - _logger.LogInformation("Try saving {0} ItemValues entries.", dbContext.ItemValues.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People ItemValues took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + foreach (var entity in queryResult) + { + var userData = GetUserData(users, entity, userIdBlacklist); + if (userData is null) + { + var userDataId = entity.GetString(0); + var internalUserId = entity.GetInt32(1); - _logger.LogInformation("Start moving UserData."); - var queryResult = connection.Query(""" - SELECT key, userId, rating, played, playCount, isFavorite, playbackPositionTicks, lastPlayedDate, AudioStreamIndex, SubtitleStreamIndex FROM UserDatas + if (!userIdBlacklist.Contains(internalUserId)) + { + _logger.LogError("Was not able to migrate user data with key {0} because its id {InternalId} does not match any existing user.", userDataId, internalUserId); + userIdBlacklist.Add(internalUserId); + } - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.UserDataKey = UserDatas.key) - """); + continue; + } - dbContext.UserData.ExecuteDelete(); + if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + { + _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); + continue; + } - var users = dbContext.Users.AsNoTracking().ToImmutableArray(); + userData.ItemId = refItem.Id; + operation.JellyfinDbContext.UserData.Add(userData); + } - foreach (var entity in queryResult) - { - var userData = GetUserData(users, entity); - if (userData is null) - { - _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0)); - continue; + users.Clear(); } - if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem)) + legacyBaseItemWithUserKeys.Clear(); + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.UserData.Local.Count} UserData entries", _logger)) { - _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0)); - continue; + operation.JellyfinDbContext.SaveChanges(); } - - userData.ItemId = refItem.Id; - dbContext.UserData.Add(userData); } - users.Clear(); - legacyBaseItemWithUserKeys.Clear(); - _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count); - dbContext.SaveChanges(); - - _logger.LogInformation("Start moving MediaStreamInfos."); - const string mediaStreamQuery = """ - SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, - IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, - AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, - Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, - DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired - FROM MediaStreams - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) - """; - dbContext.MediaStreamInfos.ExecuteDelete(); - - foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + using (var operation = GetPreparedDbContext("moving MediaStreamInfos")) { - dbContext.MediaStreamInfos.Add(GetMediaStream(dto)); - } - - _logger.LogInformation("Try saving {0} MediaStreamInfos entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); + const string mediaStreamQuery = + """ + SELECT ItemId, StreamIndex, StreamType, Codec, Language, ChannelLayout, Profile, AspectRatio, Path, + IsInterlaced, BitRate, Channels, SampleRate, IsDefault, IsForced, IsExternal, Height, Width, + AverageFrameRate, RealFrameRate, Level, PixelFormat, BitDepth, IsAnamorphic, RefFrames, CodecTag, + Comment, NalLengthSize, IsAvc, Title, TimeBase, CodecTimeBase, ColorPrimaries, ColorSpace, ColorTransfer, + DvVersionMajor, DvVersionMinor, DvProfile, DvLevel, RpuPresentFlag, ElPresentFlag, BlPresentFlag, DvBlSignalCompatibilityId, IsHearingImpaired + FROM MediaStreams + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = MediaStreams.ItemId) + """; - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving MediaStreamInfos entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); - - _logger.LogInformation("Start moving People."); - const string personsQuery = """ - SELECT ItemId, Name, Role, PersonType, SortOrder FROM People - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) - """; - dbContext.Peoples.ExecuteDelete(); - dbContext.PeopleBaseItemMap.ExecuteDelete(); - - var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - var baseItemIds = dbContext.BaseItems.Select(b => b.Id).ToHashSet(); + using (new TrackedMigrationStep("loading MediaStreamInfos", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(mediaStreamQuery)) + { + operation.JellyfinDbContext.MediaStreamInfos.Add(GetMediaStream(dto)); + } + } - foreach (SqliteDataReader reader in connection.Query(personsQuery)) - { - var itemId = reader.GetGuid(0); - if (!baseItemIds.Contains(itemId)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.MediaStreamInfos.Local.Count} MediaStreamInfos entries", _logger)) { - _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); - continue; + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving AttachmentStreamInfos")) + { + const string mediaAttachmentQuery = + """ + SELECT ItemId, AttachmentIndex, Codec, CodecTag, Comment, filename, MIMEType + FROM mediaattachments + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = mediaattachments.ItemId) + """; - var entity = GetPerson(reader); - if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + using (new TrackedMigrationStep("loading AttachmentStreamInfos", _logger)) { - peopleCache[entity.Name] = personCache = (entity, []); + foreach (SqliteDataReader dto in connection.Query(mediaAttachmentQuery)) + { + operation.JellyfinDbContext.AttachmentStreamInfos.Add(GetMediaAttachment(dto)); + } } - if (reader.TryGetString(2, out var role)) + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AttachmentStreamInfos.Local.Count} AttachmentStreamInfos entries", _logger)) { + operation.JellyfinDbContext.SaveChanges(); } + } + + using (var operation = GetPreparedDbContext("moving People")) + { + const string personsQuery = + """ + SELECT ItemId, Name, Role, PersonType, SortOrder FROM People + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = People.ItemId) + """; - int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); + var peopleCache = new Dictionary<string, (People Person, List<PeopleBaseItemMap> Items)>(); - personCache.Items.Add(new PeopleBaseItemMap() + using (new TrackedMigrationStep("loading People", _logger)) { - Item = null!, - ItemId = itemId, - People = null!, - PeopleId = personCache.Person.Id, - ListOrder = sortOrder, - SortOrder = sortOrder, - Role = role - }); - } + foreach (SqliteDataReader reader in connection.Query(personsQuery)) + { + var itemId = reader.GetGuid(0); + if (!baseItemIds.Contains(itemId)) + { + _logger.LogError("Dont save person {0} because its not in use by any BaseItem", reader.GetString(1)); + continue; + } - baseItemIds.Clear(); + var entity = GetPerson(reader); + if (!peopleCache.TryGetValue(entity.Name, out var personCache)) + { + peopleCache[entity.Name] = personCache = (entity, []); + } - foreach (var item in peopleCache) - { - dbContext.Peoples.Add(item.Value.Person); - dbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); - } + if (reader.TryGetString(2, out var role)) + { + } - peopleCache.Clear(); + int? sortOrder = reader.IsDBNull(4) ? null : reader.GetInt32(4); - _logger.LogInformation("Try saving {0} People entries.", dbContext.MediaStreamInfos.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving People entries took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + personCache.Items.Add(new PeopleBaseItemMap() + { + Item = null!, + ItemId = itemId, + People = null!, + PeopleId = personCache.Person.Id, + ListOrder = sortOrder, + SortOrder = sortOrder, + Role = role + }); + } - _logger.LogInformation("Start moving Chapters."); - const string chapterQuery = """ - SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 - WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) - """; - dbContext.Chapters.ExecuteDelete(); + baseItemIds.Clear(); - foreach (SqliteDataReader dto in connection.Query(chapterQuery)) - { - var chapter = GetChapter(dto); - dbContext.Chapters.Add(chapter); - } + foreach (var item in peopleCache) + { + operation.JellyfinDbContext.Peoples.Add(item.Value.Person); + operation.JellyfinDbContext.PeopleBaseItemMap.AddRange(item.Value.Items.DistinctBy(e => (e.ItemId, e.PeopleId))); + } - _logger.LogInformation("Try saving {0} Chapters entries.", dbContext.Chapters.Local.Count); - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving Chapters took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + peopleCache.Clear(); + } - _logger.LogInformation("Start moving AncestorIds."); - const string ancestorIdsQuery = """ - SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds - WHERE - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) - AND - EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) - """; - dbContext.Chapters.ExecuteDelete(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Peoples.Local.Count} People entries and {operation.JellyfinDbContext.PeopleBaseItemMap.Local.Count} maps", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } - foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + using (var operation = GetPreparedDbContext("moving Chapters")) { - var ancestorId = GetAncestorId(dto); - dbContext.AncestorIds.Add(ancestorId); + const string chapterQuery = + """ + SELECT ItemId,StartPositionTicks,Name,ImagePath,ImageDateModified,ChapterIndex from Chapters2 + WHERE EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = Chapters2.ItemId) + """; + + using (new TrackedMigrationStep("loading Chapters", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(chapterQuery)) + { + var chapter = GetChapter(dto); + operation.JellyfinDbContext.Chapters.Add(chapter); + } + } + + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.Chapters.Local.Count} Chapters entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } } - _logger.LogInformation("Try saving {0} AncestorIds entries.", dbContext.Chapters.Local.Count); + using (var operation = GetPreparedDbContext("moving AncestorIds")) + { + const string ancestorIdsQuery = + """ + SELECT ItemId, AncestorId, AncestorIdText FROM AncestorIds + WHERE + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.ItemId) + AND + EXISTS(SELECT 1 FROM TypedBaseItems WHERE TypedBaseItems.guid = AncestorIds.AncestorId) + """; + + using (new TrackedMigrationStep("loading AncestorIds", _logger)) + { + foreach (SqliteDataReader dto in connection.Query(ancestorIdsQuery)) + { + var ancestorId = GetAncestorId(dto); + operation.JellyfinDbContext.AncestorIds.Add(ancestorId); + } + } - dbContext.SaveChanges(); - migrationTotalTime += stopwatch.Elapsed; - _logger.LogInformation("Saving AncestorIds took {0}.", stopwatch.Elapsed); - stopwatch.Restart(); + using (new TrackedMigrationStep($"saving {operation.JellyfinDbContext.AncestorIds.Local.Count} AncestorId entries", _logger)) + { + operation.JellyfinDbContext.SaveChanges(); + } + } connection.Close(); + _logger.LogInformation("Migration of the Library.db done."); - _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); + _logger.LogInformation("Migrating Library db took {0}.", fullOperationTimer.Elapsed); SqliteConnection.ClearAllPools(); + _logger.LogInformation("Move {0} to {1}.", libraryDbPath, libraryDbPath + ".old"); File.Move(libraryDbPath, libraryDbPath + ".old", true); - _logger.LogInformation("Migrating Library db took {0}.", migrationTotalTime); - _jellyfinDatabaseProvider.RunScheduledOptimisation(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); } - private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto) + private DatabaseMigrationStep GetPreparedDbContext(string operationName) + { + var dbContext = _provider.CreateDbContext(); + dbContext.ChangeTracker.AutoDetectChangesEnabled = false; + dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; + return new DatabaseMigrationStep(dbContext, operationName, _logger); + } + + private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto, HashSet<int> userIdBlacklist) { var internalUserId = dto.GetInt32(1); var user = users.FirstOrDefault(e => e.InternalId == internalUserId); if (user is null) { + if (userIdBlacklist.Contains(internalUserId)) + { + return null; + } + _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", internalUserId, users.Length); return null; } @@ -654,6 +733,48 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return item; } + /// <summary> + /// Gets the attachment. + /// </summary> + /// <param name="reader">The reader.</param> + /// <returns>MediaAttachment.</returns> + private AttachmentStreamInfo GetMediaAttachment(SqliteDataReader reader) + { + var item = new AttachmentStreamInfo + { + Index = reader.GetInt32(1), + Item = null!, + ItemId = reader.GetGuid(0), + }; + + if (reader.TryGetString(2, out var codec)) + { + item.Codec = codec; + } + + if (reader.TryGetString(3, out var codecTag)) + { + item.CodecTag = codecTag; + } + + if (reader.TryGetString(4, out var comment)) + { + item.Comment = comment; + } + + if (reader.TryGetString(5, out var fileName)) + { + item.Filename = fileName; + } + + if (reader.TryGetString(6, out var mimeType)) + { + item.MimeType = mimeType; + } + + return item; + } + private (BaseItemEntity BaseItem, string[] LegacyUserDataKey) GetItem(SqliteDataReader reader) { var entity = new BaseItemEntity() @@ -1214,4 +1335,58 @@ internal class MigrateLibraryDb : IDatabaseMigrationRoutine return image; } + + private class TrackedMigrationStep : IDisposable + { + private readonly string _operationName; + private readonly ILogger _logger; + private readonly Stopwatch _operationTimer; + private bool _disposed; + + public TrackedMigrationStep(string operationName, ILogger logger) + { + _operationName = operationName; + _logger = logger; + _operationTimer = Stopwatch.StartNew(); + logger.LogInformation("Start {OperationName}", operationName); + } + + public bool Disposed + { + get => _disposed; + set => _disposed = value; + } + + public virtual void Dispose() + { + if (Disposed) + { + return; + } + + Disposed = true; + _logger.LogInformation("{OperationName} took '{Time}'", _operationName, _operationTimer.Elapsed); + } + } + + private sealed class DatabaseMigrationStep : TrackedMigrationStep + { + public DatabaseMigrationStep(JellyfinDbContext jellyfinDbContext, string operationName, ILogger logger) : base(operationName, logger) + { + JellyfinDbContext = jellyfinDbContext; + } + + public JellyfinDbContext JellyfinDbContext { get; } + + public override void Dispose() + { + if (Disposed) + { + return; + } + + JellyfinDbContext.Dispose(); + base.Dispose(); + } + } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs index 9c2184029..c38beb723 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateRatingLevels.cs @@ -1,36 +1,33 @@ using System; -using System.Globalization; -using System.IO; -using Emby.Server.Implementations.Data; -using MediaBrowser.Controller; +using System.Linq; +using Jellyfin.Database.Implementations; using MediaBrowser.Model.Globalization; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines { /// <summary> - /// Migrate rating levels to new rating level system. + /// Migrate rating levels. /// </summary> - internal class MigrateRatingLevels : IMigrationRoutine + internal class MigrateRatingLevels : IDatabaseMigrationRoutine { - private const string DbFilename = "library.db"; private readonly ILogger<MigrateRatingLevels> _logger; - private readonly IServerApplicationPaths _applicationPaths; + private readonly IDbContextFactory<JellyfinDbContext> _provider; private readonly ILocalizationManager _localizationManager; public MigrateRatingLevels( - IServerApplicationPaths applicationPaths, + IDbContextFactory<JellyfinDbContext> provider, ILoggerFactory loggerFactory, ILocalizationManager localizationManager) { - _applicationPaths = applicationPaths; + _provider = provider; _localizationManager = localizationManager; _logger = loggerFactory.CreateLogger<MigrateRatingLevels>(); } /// <inheritdoc/> - public Guid Id => Guid.Parse("{73DAB92A-178B-48CD-B05B-FE18733ACDC8}"); + public Guid Id => Guid.Parse("{98724538-EB11-40E3-931A-252C55BDDE7A}"); /// <inheritdoc/> public string Name => "MigrateRatingLevels"; @@ -41,54 +38,37 @@ namespace Jellyfin.Server.Migrations.Routines /// <inheritdoc/> public void Perform() { - var dbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); - - // Back up the database before modifying any entries - for (int i = 1; ; i++) + _logger.LogInformation("Recalculating parental rating levels based on rating string."); + using var context = _provider.CreateDbContext(); + using var transaction = context.Database.BeginTransaction(); + var ratings = context.BaseItems.AsNoTracking().Select(e => e.OfficialRating).Distinct(); + foreach (var rating in ratings) { - var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", dbPath, i); - if (!File.Exists(bakPath)) + if (string.IsNullOrEmpty(rating)) { - try - { - File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); - throw; - } + int? value = null; + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, value)); + context.BaseItems + .Where(e => e.OfficialRating == null || e.OfficialRating == string.Empty) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, value)); } - } - - // Migrate parental rating strings to new levels - _logger.LogInformation("Recalculating parental rating levels based on rating string."); - using var connection = new SqliteConnection($"Filename={dbPath}"); - connection.Open(); - using (var transaction = connection.BeginTransaction()) - { - var queryResult = connection.Query("SELECT DISTINCT OfficialRating FROM TypedBaseItems"); - foreach (var entry in queryResult) + else { - if (!entry.TryGetString(0, out var ratingString) || string.IsNullOrEmpty(ratingString)) - { - connection.Execute("UPDATE TypedBaseItems SET InheritedParentalRatingValue = NULL WHERE OfficialRating IS NULL OR OfficialRating='';"); - } - else - { - var ratingValue = _localizationManager.GetRatingLevel(ratingString)?.ToString(CultureInfo.InvariantCulture) ?? "NULL"; - - using var statement = connection.PrepareStatement("UPDATE TypedBaseItems SET InheritedParentalRatingValue = @Value WHERE OfficialRating = @Rating;"); - statement.TryBind("@Value", ratingValue); - statement.TryBind("@Rating", ratingString); - statement.ExecuteNonQuery(); - } + var ratingValue = _localizationManager.GetRatingScore(rating); + var score = ratingValue?.Score; + var subScore = ratingValue?.SubScore; + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingValue, score)); + context.BaseItems + .Where(e => e.OfficialRating == rating) + .ExecuteUpdate(f => f.SetProperty(e => e.InheritedParentalRatingSubValue, subScore)); } - - transaction.Commit(); } + + transaction.Commit(); } } } diff --git a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs index c40560660..1b5fab7a8 100644 --- a/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs +++ b/Jellyfin.Server/Migrations/Routines/MigrateUserDb.cs @@ -112,7 +112,8 @@ namespace Jellyfin.Server.Migrations.Routines { Id = entry.GetGuid(1), InternalId = entry.GetInt64(0), - MaxParentalAgeRating = policy.MaxParentalRating, + MaxParentalRatingScore = policy.MaxParentalRating, + MaxParentalRatingSubScore = null, EnableUserPreferenceAccess = policy.EnableUserPreferenceAccess, RemoteClientBitrateLimit = policy.RemoteClientBitrateLimit, InvalidLoginAttemptCount = policy.InvalidLoginAttemptCount, diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs new file mode 100644 index 000000000..f63c5fd40 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs @@ -0,0 +1,299 @@ +#pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Jellyfin.Data.Enums; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.MediaInfo; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// <summary> +/// Migration to move extracted files to the new directories. +/// </summary> +public class MoveExtractedFiles : IDatabaseMigrationRoutine +{ + private readonly IApplicationPaths _appPaths; + private readonly ILibraryManager _libraryManager; + private readonly ILogger<MoveExtractedFiles> _logger; + private readonly IMediaSourceManager _mediaSourceManager; + private readonly IPathManager _pathManager; + + /// <summary> + /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class. + /// </summary> + /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> + /// <param name="logger">The logger.</param> + /// <param name="mediaSourceManager">Instance of the <see cref="IMediaSourceManager"/> interface.</param> + /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param> + public MoveExtractedFiles( + IApplicationPaths appPaths, + ILibraryManager libraryManager, + ILogger<MoveExtractedFiles> logger, + IMediaSourceManager mediaSourceManager, + IPathManager pathManager) + { + _appPaths = appPaths; + _libraryManager = libraryManager; + _logger = logger; + _mediaSourceManager = mediaSourceManager; + _pathManager = pathManager; + } + + private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles"); + + private string AttachmentCachePath => Path.Combine(_appPaths.DataPath, "attachments"); + + /// <inheritdoc /> + public Guid Id => new("9063b0Ef-CFF1-4EDC-9A13-74093681A89B"); + + /// <inheritdoc /> + public string Name => "MoveExtractedFiles"; + + /// <inheritdoc /> + public bool PerformOnNewInstall => false; + + /// <inheritdoc /> + public void Perform() + { + const int Limit = 500; + int itemCount = 0, offset = 0; + + var sw = Stopwatch.StartNew(); + var itemsQuery = new InternalItemsQuery + { + MediaTypes = [MediaType.Video], + SourceTypes = [SourceType.Library], + IsVirtualItem = false, + IsFolder = false, + Limit = Limit, + StartIndex = offset, + EnableTotalRecordCount = true, + }; + + var records = _libraryManager.GetItemsResult(itemsQuery).TotalRecordCount; + _logger.LogInformation("Checking {Count} items for movable extracted files.", records); + + // Make sure directories exist + Directory.CreateDirectory(SubtitleCachePath); + Directory.CreateDirectory(AttachmentCachePath); + + itemsQuery.EnableTotalRecordCount = false; + do + { + itemsQuery.StartIndex = offset; + var result = _libraryManager.GetItemsResult(itemsQuery); + + var items = result.Items; + foreach (var item in items) + { + if (MoveSubtitleAndAttachmentFiles(item)) + { + itemCount++; + } + } + + offset += Limit; + if (offset % 5_000 == 0) + { + _logger.LogInformation("Checked extracted files for {Count} items in {Time}.", offset, sw.Elapsed); + } + } while (offset < records); + + _logger.LogInformation("Checked {Checked} items - Moved files for {Items} items in {Time}.", records, itemCount, sw.Elapsed); + + // Get all subdirectories with 1 character names (those are the legacy directories) + var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList(); + subdirectories.AddRange(Directory.GetDirectories(AttachmentCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == AttachmentCachePath.Length + 2)); + + // Remove all legacy subdirectories + foreach (var subdir in subdirectories) + { + Directory.Delete(subdir, true); + } + + // Remove old cache path + var attachmentCachePath = Path.Join(_appPaths.CachePath, "attachments"); + if (Directory.Exists(attachmentCachePath)) + { + Directory.Delete(attachmentCachePath, true); + } + + _logger.LogInformation("Cleaned up left over subtitles and attachments."); + } + + private bool MoveSubtitleAndAttachmentFiles(BaseItem item) + { + var mediaStreams = item.GetMediaStreams().Where(s => s.Type == MediaStreamType.Subtitle && !s.IsExternal); + var itemIdString = item.Id.ToString("N", CultureInfo.InvariantCulture); + var modified = false; + foreach (var mediaStream in mediaStreams) + { + if (mediaStream.Codec is null) + { + continue; + } + + var mediaStreamIndex = mediaStream.Index; + var extension = GetSubtitleExtension(mediaStream.Codec); + var oldSubtitleCachePath = GetOldSubtitleCachePath(item.Path, mediaStream.Index, extension); + if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath)) + { + continue; + } + + var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension); + if (File.Exists(newSubtitleCachePath)) + { + File.Delete(oldSubtitleCachePath); + } + else + { + var newDirectory = Path.GetDirectoryName(newSubtitleCachePath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldSubtitleCachePath, newSubtitleCachePath, false); + _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, item.Id, oldSubtitleCachePath, newSubtitleCachePath); + + modified = true; + } + } + } + + var attachments = _mediaSourceManager.GetMediaAttachments(item.Id).Where(a => !string.Equals(a.Codec, "mjpeg", StringComparison.OrdinalIgnoreCase)).ToList(); + var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.FileName) + && (a.FileName.Contains('/', StringComparison.OrdinalIgnoreCase) || a.FileName.Contains('\\', StringComparison.OrdinalIgnoreCase))); + foreach (var attachment in attachments) + { + var attachmentIndex = attachment.Index; + var oldAttachmentPath = GetOldAttachmentDataPath(item.Path, attachmentIndex); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne); + if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath)) + { + continue; + } + } + + var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.FileName ?? attachmentIndex.ToString(CultureInfo.InvariantCulture)); + if (File.Exists(newAttachmentPath)) + { + File.Delete(oldAttachmentPath); + } + else + { + var newDirectory = Path.GetDirectoryName(newAttachmentPath); + if (newDirectory is not null) + { + Directory.CreateDirectory(newDirectory); + File.Move(oldAttachmentPath, newAttachmentPath, false); + _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, item.Id, oldAttachmentPath, newAttachmentPath); + + modified = true; + } + } + } + + return modified; + } + + private string? GetOldAttachmentDataPath(string? mediaPath, int attachmentStreamIndex) + { + if (mediaPath is null) + { + return null; + } + + string filename; + var protocol = _mediaSourceManager.GetPathProtocol(mediaPath); + if (protocol == MediaProtocol.File) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(mediaPath); + } + catch (IOException e) + { + _logger.LogDebug("Skipping attachment at index {Index} for {Path}: {Exception}", attachmentStreamIndex, mediaPath, e.Message); + + return null; + } + + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + else + { + filename = (mediaPath + attachmentStreamIndex.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("D", CultureInfo.InvariantCulture); + } + + return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename); + } + + private string? GetOldAttachmentCachePath(string mediaSourceId, MediaAttachment attachment, bool shouldExtractOneByOne) + { + var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId); + if (shouldExtractOneByOne) + { + return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture)); + } + + if (string.IsNullOrEmpty(attachment.FileName)) + { + return null; + } + + return Path.Join(attachmentFolderPath, attachment.FileName); + } + + private string? GetOldSubtitleCachePath(string path, int streamIndex, string outputSubtitleExtension) + { + DateTime? date; + try + { + date = File.GetLastWriteTimeUtc(path); + } + catch (IOException e) + { + _logger.LogDebug("Skipping subtitle at index {Index} for {Path}: {Exception}", streamIndex, path, e.Message); + + return null; + } + + var ticksParam = string.Empty; + ReadOnlySpan<char> filename = new Guid(MD5.HashData(Encoding.Unicode.GetBytes(path + "_" + streamIndex.ToString(CultureInfo.InvariantCulture) + "_" + date.Value.Ticks.ToString(CultureInfo.InvariantCulture) + ticksParam))) + outputSubtitleExtension; + + return Path.Join(SubtitleCachePath, filename[..1], filename); + } + + private static string GetSubtitleExtension(string codec) + { + if (codec.ToLower(CultureInfo.InvariantCulture).Equals("ass", StringComparison.OrdinalIgnoreCase) + || codec.ToLower(CultureInfo.InvariantCulture).Equals("ssa", StringComparison.OrdinalIgnoreCase)) + { + return "." + codec; + } + else if (codec.Contains("pgs", StringComparison.OrdinalIgnoreCase)) + { + return ".sup"; + } + else + { + return ".srt"; + } + } +} diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs index f4ebac377..eeb11e14c 100644 --- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs +++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs @@ -4,7 +4,6 @@ using System.Globalization; using System.IO; using System.Linq; using Jellyfin.Data.Enums; -using MediaBrowser.Common; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Trickplay; @@ -16,7 +15,7 @@ namespace Jellyfin.Server.Migrations.Routines; /// <summary> /// Migration to move trickplay files to the new directory. /// </summary> -public class MoveTrickplayFiles : IMigrationRoutine +public class MoveTrickplayFiles : IDatabaseMigrationRoutine { private readonly ITrickplayManager _trickplayManager; private readonly IFileSystem _fileSystem; diff --git a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs index f84bccc25..e183a1d63 100644 --- a/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs +++ b/Jellyfin.Server/Migrations/Routines/RemoveDuplicatePlaylistChildren.cs @@ -1,12 +1,10 @@ using System; using System.Linq; using System.Threading; - using Jellyfin.Data.Enums; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Playlists; -using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations.Routines; @@ -15,16 +13,13 @@ namespace Jellyfin.Server.Migrations.Routines; /// </summary> internal class RemoveDuplicatePlaylistChildren : IMigrationRoutine { - private readonly ILogger<RemoveDuplicatePlaylistChildren> _logger; private readonly ILibraryManager _libraryManager; private readonly IPlaylistManager _playlistManager; public RemoveDuplicatePlaylistChildren( - ILogger<RemoveDuplicatePlaylistChildren> logger, ILibraryManager libraryManager, IPlaylistManager playlistManager) { - _logger = logger; _libraryManager = libraryManager; _playlistManager = playlistManager; } |