Optimize migrations (#13855)

3 files changed, 118 insertions, 131 deletions

diff --git a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs
index b8e69db8e..68d7a7b87 100644
--- a/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs
+++ b/Jellyfin.Server/Migrations/Routines/MigrateKeyframeData.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
@@ -12,8 +11,6 @@ using Jellyfin.Database.Implementations.Entities;
 using Jellyfin.Extensions.Json;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
-using MediaBrowser.Controller.Entities;
-using MediaBrowser.Controller.Library;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
@@ -24,8 +21,7 @@ namespace Jellyfin.Server.Migrations.Routines;
 /// </summary>
 public class MigrateKeyframeData : IDatabaseMigrationRoutine
 {
-    private readonly ILibraryManager _libraryManager;
-    private readonly ILogger<MoveTrickplayFiles> _logger;
+    private readonly ILogger<MigrateKeyframeData> _logger;
     private readonly IApplicationPaths _appPaths;
     private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
     private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options;
@@ -33,17 +29,14 @@ public class MigrateKeyframeData : IDatabaseMigrationRoutine
     /// <summary>
     /// Initializes a new instance of the <see cref="MigrateKeyframeData"/> class.
     /// </summary>
-    /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param>
     /// <param name="logger">The logger.</param>
     /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param>
     /// <param name="dbProvider">The EFCore db factory.</param>
     public MigrateKeyframeData(
-        ILibraryManager libraryManager,
-        ILogger<MoveTrickplayFiles> logger,
+        ILogger<MigrateKeyframeData> logger,
         IApplicationPaths appPaths,
         IDbContextFactory<JellyfinDbContext> dbProvider)
     {
-        _libraryManager = libraryManager;
         _logger = logger;
         _appPaths = appPaths;
         _dbProvider = dbProvider;
@@ -63,48 +56,34 @@ public class MigrateKeyframeData : IDatabaseMigrationRoutine
     /// <inheritdoc />
     public void Perform()
     {
-        const int Limit = 100;
-        int itemCount = 0, offset = 0, previousCount;
+        const int Limit = 5000;
+        int itemCount = 0, offset = 0;
 
         var sw = Stopwatch.StartNew();
-        var itemsQuery = new InternalItemsQuery
-        {
-            MediaTypes = [MediaType.Video],
-            SourceTypes = [SourceType.Library],
-            IsVirtualItem = false,
-            IsFolder = false
-        };
 
         using var context = _dbProvider.CreateDbContext();
+        var baseQuery = context.BaseItems.Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder).OrderBy(e => e.Id);
+        var records = baseQuery.Count();
+        _logger.LogInformation("Checking {Count} items for importable keyframe data.", records);
+
         context.KeyframeData.ExecuteDelete();
         using var transaction = context.Database.BeginTransaction();
-        List<KeyframeData> keyframes = [];
-
         do
         {
-            var result = _libraryManager.GetItemsResult(itemsQuery);
-            _logger.LogInformation("Importing keyframes for {Count} items", result.TotalRecordCount);
-
-            var items = result.Items;
-            previousCount = items.Count;
-            offset += Limit;
-            foreach (var item in items)
+            var results = baseQuery.Skip(offset).Take(Limit).Select(b => new Tuple<Guid, string?>(b.Id, b.Path)).ToList();
+            foreach (var result in results)
             {
-                if (TryGetKeyframeData(item, out var data))
+                if (TryGetKeyframeData(result.Item1, result.Item2, out var data))
                 {
-                    keyframes.Add(data);
-                }
-
-                if (++itemCount % 10_000 == 0)
-                {
-                    context.KeyframeData.AddRange(keyframes);
-                    keyframes.Clear();
-                    _logger.LogInformation("Imported keyframes for {Count} items in {Time}", itemCount, sw.Elapsed);
+                    itemCount++;
+                    context.KeyframeData.Add(data);
                 }
             }
-        } while (previousCount == Limit);
 
-        context.KeyframeData.AddRange(keyframes);
+            offset += Limit;
+            _logger.LogInformation("Checked: {Count} - Imported: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed);
+        } while (offset < records);
+
         context.SaveChanges();
         transaction.Commit();
 
@@ -116,10 +95,9 @@ public class MigrateKeyframeData : IDatabaseMigrationRoutine
         }
     }
 
-    private bool TryGetKeyframeData(BaseItem item, [NotNullWhen(true)] out KeyframeData? data)
+    private bool TryGetKeyframeData(Guid id, string? path, [NotNullWhen(true)] out KeyframeData? data)
     {
         data = null;
-        var path = item.Path;
         if (!string.IsNullOrEmpty(path))
         {
             var cachePath = GetCachePath(KeyframeCachePath, path);
@@ -127,7 +105,7 @@ public class MigrateKeyframeData : IDatabaseMigrationRoutine
             {
                 data = new()
                 {
-                    ItemId = item.Id,
+                    ItemId = id,
                     KeyframeTicks = keyframeData.KeyframeTicks.ToList(),
                     TotalDuration = keyframeData.TotalDuration
                 };
diff --git a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs
index f63c5fd40..c5bbcd6f9 100644
--- a/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs
+++ b/Jellyfin.Server/Migrations/Routines/MoveExtractedFiles.cs
@@ -1,6 +1,7 @@
 #pragma warning disable CA5351 // Do Not Use Broken Cryptographic Algorithms
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -8,13 +9,14 @@ using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
 using Jellyfin.Data.Enums;
+using Jellyfin.Database.Implementations;
+using Jellyfin.Database.Implementations.Entities;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
-using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.IO;
-using MediaBrowser.Controller.Library;
 using MediaBrowser.Model.Entities;
-using MediaBrowser.Model.MediaInfo;
+using MediaBrowser.Model.IO;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
 namespace Jellyfin.Server.Migrations.Routines;
@@ -22,34 +24,34 @@ namespace Jellyfin.Server.Migrations.Routines;
 /// <summary>
 /// Migration to move extracted files to the new directories.
 /// </summary>
-public class MoveExtractedFiles : IDatabaseMigrationRoutine
+public class MoveExtractedFiles : IMigrationRoutine
 {
     private readonly IApplicationPaths _appPaths;
-    private readonly ILibraryManager _libraryManager;
     private readonly ILogger<MoveExtractedFiles> _logger;
-    private readonly IMediaSourceManager _mediaSourceManager;
+    private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
     private readonly IPathManager _pathManager;
+    private readonly IFileSystem _fileSystem;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="MoveExtractedFiles"/> class.
     /// </summary>
     /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param>
-    /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param>
     /// <param name="logger">The logger.</param>
-    /// <param name="mediaSourceManager">Instance of the <see cref="IMediaSourceManager"/> interface.</param>
+    /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param>
     /// <param name="pathManager">Instance of the <see cref="IPathManager"/> interface.</param>
+    /// <param name="dbProvider">Instance of the <see cref="IDbContextFactory{JellyfinDbContext}"/> interface.</param>
     public MoveExtractedFiles(
         IApplicationPaths appPaths,
-        ILibraryManager libraryManager,
         ILogger<MoveExtractedFiles> logger,
-        IMediaSourceManager mediaSourceManager,
-        IPathManager pathManager)
+        IPathManager pathManager,
+        IFileSystem fileSystem,
+        IDbContextFactory<JellyfinDbContext> dbProvider)
     {
         _appPaths = appPaths;
-        _libraryManager = libraryManager;
         _logger = logger;
-        _mediaSourceManager = mediaSourceManager;
         _pathManager = pathManager;
+        _fileSystem = fileSystem;
+        _dbProvider = dbProvider;
     }
 
     private string SubtitleCachePath => Path.Combine(_appPaths.DataPath, "subtitles");
@@ -68,51 +70,41 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
     /// <inheritdoc />
     public void Perform()
     {
-        const int Limit = 500;
+        const int Limit = 5000;
         int itemCount = 0, offset = 0;
 
         var sw = Stopwatch.StartNew();
-        var itemsQuery = new InternalItemsQuery
-        {
-            MediaTypes = [MediaType.Video],
-            SourceTypes = [SourceType.Library],
-            IsVirtualItem = false,
-            IsFolder = false,
-            Limit = Limit,
-            StartIndex = offset,
-            EnableTotalRecordCount = true,
-        };
-
-        var records = _libraryManager.GetItemsResult(itemsQuery).TotalRecordCount;
+
+        using var context = _dbProvider.CreateDbContext();
+        var records = context.BaseItems.Count(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder);
         _logger.LogInformation("Checking {Count} items for movable extracted files.", records);
 
         // Make sure directories exist
         Directory.CreateDirectory(SubtitleCachePath);
         Directory.CreateDirectory(AttachmentCachePath);
-
-        itemsQuery.EnableTotalRecordCount = false;
         do
         {
-            itemsQuery.StartIndex = offset;
-            var result = _libraryManager.GetItemsResult(itemsQuery);
-
-            var items = result.Items;
-            foreach (var item in items)
+            var results = context.BaseItems
+                            .Include(e => e.MediaStreams!.Where(s => s.StreamType == MediaStreamTypeEntity.Subtitle && !s.IsExternal))
+                            .Where(b => b.MediaType == MediaType.Video.ToString() && !b.IsVirtualItem && !b.IsFolder)
+                            .OrderBy(e => e.Id)
+                            .Skip(offset)
+                            .Take(Limit)
+                            .Select(b => new Tuple<Guid, string?, ICollection<MediaStreamInfo>?>(b.Id, b.Path, b.MediaStreams)).ToList();
+
+            foreach (var result in results)
             {
-                if (MoveSubtitleAndAttachmentFiles(item))
+                if (MoveSubtitleAndAttachmentFiles(result.Item1, result.Item2, result.Item3, context))
                 {
                     itemCount++;
                 }
             }
 
             offset += Limit;
-            if (offset % 5_000 == 0)
-            {
-                _logger.LogInformation("Checked extracted files for {Count} items in {Time}.", offset, sw.Elapsed);
-            }
+            _logger.LogInformation("Checked: {Count} - Moved: {Items} - Time: {Time}", offset, itemCount, sw.Elapsed);
         } while (offset < records);
 
-        _logger.LogInformation("Checked {Checked} items - Moved files for {Items} items in {Time}.", records, itemCount, sw.Elapsed);
+        _logger.LogInformation("Moved files for {Count} items in {Time}", itemCount, sw.Elapsed);
 
         // Get all subdirectories with 1 character names (those are the legacy directories)
         var subdirectories = Directory.GetDirectories(SubtitleCachePath, "*", SearchOption.AllDirectories).Where(s => s.Length == SubtitleCachePath.Length + 2).ToList();
@@ -134,52 +126,56 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
         _logger.LogInformation("Cleaned up left over subtitles and attachments.");
     }
 
-    private bool MoveSubtitleAndAttachmentFiles(BaseItem item)
+    private bool MoveSubtitleAndAttachmentFiles(Guid id, string? path, ICollection<MediaStreamInfo>? mediaStreams, JellyfinDbContext context)
     {
-        var mediaStreams = item.GetMediaStreams().Where(s => s.Type == MediaStreamType.Subtitle && !s.IsExternal);
-        var itemIdString = item.Id.ToString("N", CultureInfo.InvariantCulture);
+        var itemIdString = id.ToString("N", CultureInfo.InvariantCulture);
         var modified = false;
-        foreach (var mediaStream in mediaStreams)
+        if (mediaStreams is not null)
         {
-            if (mediaStream.Codec is null)
+            foreach (var mediaStream in mediaStreams)
             {
-                continue;
-            }
-
-            var mediaStreamIndex = mediaStream.Index;
-            var extension = GetSubtitleExtension(mediaStream.Codec);
-            var oldSubtitleCachePath = GetOldSubtitleCachePath(item.Path, mediaStream.Index, extension);
-            if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath))
-            {
-                continue;
-            }
+                if (mediaStream.Codec is null)
+                {
+                    continue;
+                }
 
-            var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension);
-            if (File.Exists(newSubtitleCachePath))
-            {
-                File.Delete(oldSubtitleCachePath);
-            }
-            else
-            {
-                var newDirectory = Path.GetDirectoryName(newSubtitleCachePath);
-                if (newDirectory is not null)
+                var mediaStreamIndex = mediaStream.StreamIndex;
+                var extension = GetSubtitleExtension(mediaStream.Codec);
+                var oldSubtitleCachePath = GetOldSubtitleCachePath(path, mediaStreamIndex, extension);
+                if (string.IsNullOrEmpty(oldSubtitleCachePath) || !File.Exists(oldSubtitleCachePath))
                 {
-                    Directory.CreateDirectory(newDirectory);
-                    File.Move(oldSubtitleCachePath, newSubtitleCachePath, false);
-                    _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, item.Id, oldSubtitleCachePath, newSubtitleCachePath);
+                    continue;
+                }
 
-                    modified = true;
+                var newSubtitleCachePath = _pathManager.GetSubtitlePath(itemIdString, mediaStreamIndex, extension);
+                if (File.Exists(newSubtitleCachePath))
+                {
+                    File.Delete(oldSubtitleCachePath);
+                }
+                else
+                {
+                    var newDirectory = Path.GetDirectoryName(newSubtitleCachePath);
+                    if (newDirectory is not null)
+                    {
+                        Directory.CreateDirectory(newDirectory);
+                        File.Move(oldSubtitleCachePath, newSubtitleCachePath, false);
+                        _logger.LogDebug("Moved subtitle {Index} for {Item} from {Source} to {Destination}", mediaStreamIndex, id, oldSubtitleCachePath, newSubtitleCachePath);
+
+                        modified = true;
+                    }
                 }
             }
         }
 
-        var attachments = _mediaSourceManager.GetMediaAttachments(item.Id).Where(a => !string.Equals(a.Codec, "mjpeg", StringComparison.OrdinalIgnoreCase)).ToList();
-        var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.FileName)
-                                                                              && (a.FileName.Contains('/', StringComparison.OrdinalIgnoreCase) || a.FileName.Contains('\\', StringComparison.OrdinalIgnoreCase)));
+#pragma warning disable CA1309 // Use ordinal string comparison
+        var attachments = context.AttachmentStreamInfos.Where(a => a.ItemId.Equals(id) && !string.Equals(a.Codec, "mjpeg")).ToList();
+#pragma warning restore CA1309 // Use ordinal string comparison
+        var shouldExtractOneByOne = attachments.Any(a => !string.IsNullOrEmpty(a.Filename)
+                                                                              && (a.Filename.Contains('/', StringComparison.OrdinalIgnoreCase) || a.Filename.Contains('\\', StringComparison.OrdinalIgnoreCase)));
         foreach (var attachment in attachments)
         {
             var attachmentIndex = attachment.Index;
-            var oldAttachmentPath = GetOldAttachmentDataPath(item.Path, attachmentIndex);
+            var oldAttachmentPath = GetOldAttachmentDataPath(path, attachmentIndex);
             if (string.IsNullOrEmpty(oldAttachmentPath) || !File.Exists(oldAttachmentPath))
             {
                 oldAttachmentPath = GetOldAttachmentCachePath(itemIdString, attachment, shouldExtractOneByOne);
@@ -189,7 +185,7 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
                 }
             }
 
-            var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.FileName ?? attachmentIndex.ToString(CultureInfo.InvariantCulture));
+            var newAttachmentPath = _pathManager.GetAttachmentPath(itemIdString, attachment.Filename ?? attachmentIndex.ToString(CultureInfo.InvariantCulture));
             if (File.Exists(newAttachmentPath))
             {
                 File.Delete(oldAttachmentPath);
@@ -201,7 +197,7 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
                 {
                     Directory.CreateDirectory(newDirectory);
                     File.Move(oldAttachmentPath, newAttachmentPath, false);
-                    _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, item.Id, oldAttachmentPath, newAttachmentPath);
+                    _logger.LogDebug("Moved attachment {Index} for {Item} from {Source} to {Destination}", attachmentIndex, id, oldAttachmentPath, newAttachmentPath);
 
                     modified = true;
                 }
@@ -219,8 +215,7 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
         }
 
         string filename;
-        var protocol = _mediaSourceManager.GetPathProtocol(mediaPath);
-        if (protocol == MediaProtocol.File)
+        if (_fileSystem.IsPathFile(mediaPath))
         {
             DateTime? date;
             try
@@ -244,7 +239,7 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
         return Path.Join(_appPaths.DataPath, "attachments", filename[..1], filename);
     }
 
-    private string? GetOldAttachmentCachePath(string mediaSourceId, MediaAttachment attachment, bool shouldExtractOneByOne)
+    private string? GetOldAttachmentCachePath(string mediaSourceId, AttachmentStreamInfo attachment, bool shouldExtractOneByOne)
     {
         var attachmentFolderPath = Path.Join(_appPaths.CachePath, "attachments", mediaSourceId);
         if (shouldExtractOneByOne)
@@ -252,16 +247,21 @@ public class MoveExtractedFiles : IDatabaseMigrationRoutine
             return Path.Join(attachmentFolderPath, attachment.Index.ToString(CultureInfo.InvariantCulture));
         }
 
-        if (string.IsNullOrEmpty(attachment.FileName))
+        if (string.IsNullOrEmpty(attachment.Filename))
         {
             return null;
         }
 
-        return Path.Join(attachmentFolderPath, attachment.FileName);
+        return Path.Join(attachmentFolderPath, attachment.Filename);
     }
 
-    private string? GetOldSubtitleCachePath(string path, int streamIndex, string outputSubtitleExtension)
+    private string? GetOldSubtitleCachePath(string? path, int streamIndex, string outputSubtitleExtension)
     {
+        if (path is null)
+        {
+            return null;
+        }
+
         DateTime? date;
         try
         {
diff --git a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs
index eeb11e14c..a278138ce 100644
--- a/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs
+++ b/Jellyfin.Server/Migrations/Routines/MoveTrickplayFiles.cs
@@ -15,7 +15,7 @@ namespace Jellyfin.Server.Migrations.Routines;
 /// <summary>
 /// Migration to move trickplay files to the new directory.
 /// </summary>
-public class MoveTrickplayFiles : IDatabaseMigrationRoutine
+public class MoveTrickplayFiles : IMigrationRoutine
 {
     private readonly ITrickplayManager _trickplayManager;
     private readonly IFileSystem _fileSystem;
@@ -29,7 +29,11 @@ public class MoveTrickplayFiles : IDatabaseMigrationRoutine
     /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param>
     /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param>
     /// <param name="logger">The logger.</param>
-    public MoveTrickplayFiles(ITrickplayManager trickplayManager, IFileSystem fileSystem, ILibraryManager libraryManager, ILogger<MoveTrickplayFiles> logger)
+    public MoveTrickplayFiles(
+        ITrickplayManager trickplayManager,
+        IFileSystem fileSystem,
+        ILibraryManager libraryManager,
+        ILogger<MoveTrickplayFiles> logger)
     {
         _trickplayManager = trickplayManager;
         _fileSystem = fileSystem;
@@ -49,7 +53,7 @@ public class MoveTrickplayFiles : IDatabaseMigrationRoutine
     /// <inheritdoc />
     public void Perform()
     {
-        const int Limit = 100;
+        const int Limit = 5000;
         int itemCount = 0, offset = 0, previousCount;
 
         var sw = Stopwatch.StartNew();
@@ -64,9 +68,6 @@ public class MoveTrickplayFiles : IDatabaseMigrationRoutine
         do
         {
             var trickplayInfos = _trickplayManager.GetTrickplayItemsAsync(Limit, offset).GetAwaiter().GetResult();
-            previousCount = trickplayInfos.Count;
-            offset += Limit;
-
             trickplayQuery.ItemIds = trickplayInfos.Select(i => i.ItemId).Distinct().ToArray();
             var items = _libraryManager.GetItemList(trickplayQuery);
             foreach (var trickplayInfo in trickplayInfos)
@@ -77,24 +78,32 @@ public class MoveTrickplayFiles : IDatabaseMigrationRoutine
                     continue;
                 }
 
-                if (++itemCount % 1_000 == 0)
-                {
-                    _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed);
-                }
-
+                var moved = false;
                 var oldPath = GetOldTrickplayDirectory(item, trickplayInfo.Width);
                 var newPath = _trickplayManager.GetTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false);
                 if (_fileSystem.DirectoryExists(oldPath))
                 {
                     _fileSystem.MoveDirectory(oldPath, newPath);
+                    moved = true;
                 }
 
                 oldPath = GetNewOldTrickplayDirectory(item, trickplayInfo.TileWidth, trickplayInfo.TileHeight, trickplayInfo.Width, false);
                 if (_fileSystem.DirectoryExists(oldPath))
                 {
                     _fileSystem.MoveDirectory(oldPath, newPath);
+                    moved = true;
+                }
+
+                if (moved)
+                {
+                    itemCount++;
                 }
             }
+
+            offset += Limit;
+            previousCount = trickplayInfos.Count;
+
+            _logger.LogInformation("Checked: {Checked} - Moved: {Count} - Time: {Time}", itemCount, offset, sw.Elapsed);
         } while (previousCount == Limit);
 
         _logger.LogInformation("Moved {Count} items in {Time}", itemCount, sw.Elapsed);