Backport pull request #18102 from jellyfin/release-12.z

Never treat unresolvable libraries as grounds for deletion

Original-merge: a76b50a0f889a3db9c5195f375cf785c5996ff7a

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Cody Robibero <cody@robibe.ro>

2 files changed, 321 insertions, 7 deletions

diff --git a/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs b/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs
index 8f239007d8..7fa555f005 100644
--- a/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs
+++ b/Jellyfin.Server/Migrations/Routines/20260113120000_MigrateLinkedChildren.cs
@@ -9,6 +9,7 @@ using Jellyfin.Extensions;
 using MediaBrowser.Controller;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.Entities;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using LinkedChildType = Jellyfin.Database.Implementations.Entities.LinkedChildType;
@@ -26,6 +27,7 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
     private const int FileCheckProgressLogStep = 10_000;
     private const int ResolveProgressLogStep = 10_000;
     private const int DeleteProgressLogStep = 25;
+    private const string CollectionFolderTypeName = "MediaBrowser.Controller.Entities.CollectionFolder";
 
     private readonly ILogger<MigrateLinkedChildren> _logger;
     private readonly IDbContextFactory<JellyfinDbContext> _dbProvider;
@@ -415,8 +417,18 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
         // An item outside every library location is normally left over from a removed media path, but
         // it looks exactly the same as one whose storage failed to mount (a wrong bind mount on the
         // first container start, for example). Only act on it while every location is readable.
-        var canRemoveUnrootedItems = inaccessiblePaths.Count == 0;
+        // Resolving no locations at all is the same hazard with none of the evidence: it cannot be
+        // told apart from an install that has libraries the server has not read yet, and acting on it
+        // means every file-backed item in the database is outside every location.
+        var canRemoveUnrootedItems = inaccessiblePaths.Count == 0 && allLibraryPaths.Count > 0;
         var skippedUnrootedItems = 0;
+        var removedUnrootedItems = 0;
+
+        // One library losing its definition folder is the same accident confined to that library: it
+        // contributes no locations, so everything it held reads as unrooted while the other libraries
+        // still look healthy enough to act on.
+        var itemsInMissingLibraries = GetItemsInMissingLibraries(context, virtualFolders);
+        var skippedMissingLibraryItems = 0;
 
         var staleIds = new List<Guid>();
         var checkedCount = 0;
@@ -459,10 +471,16 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
             {
                 // Item is not under ANY library location (accessible or not) —
                 // it's orphaned from all libraries (e.g. media path was removed from config)
-                if (canRemoveUnrootedItems)
+                if (itemsInMissingLibraries.Contains(item.Id))
+                {
+                    // Its library is the thing that went missing, not its media path.
+                    skippedMissingLibraryItems++;
+                }
+                else if (canRemoveUnrootedItems)
                 {
                     _logger.LogDebug("Removing item {ItemId}: path {Path} is outside every library location.", item.Id, path);
                     staleIds.Add(item.Id);
+                    removedUnrootedItems++;
                 }
                 else
                 {
@@ -473,12 +491,29 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
             // Otherwise: item is under an inaccessible location — skip (storage may be offline)
         }
 
-        if (skippedUnrootedItems > 0)
+        if (skippedMissingLibraryItems > 0)
         {
             _logger.LogWarning(
-                "Keeping {Count} items that are outside every library location because {LocationCount} library location(s) are currently unavailable.",
-                skippedUnrootedItems,
-                inaccessiblePaths.Count);
+                "Keeping {Count} items whose library definition is missing from {Path}. Restore the definition or recreate the library; the items are kept meanwhile.",
+                skippedMissingLibraryItems,
+                _appPaths.DefaultUserViewsPath);
+        }
+
+        if (skippedUnrootedItems > 0)
+        {
+            if (allLibraryPaths.Count == 0)
+            {
+                _logger.LogWarning(
+                    "Keeping {Count} items that are outside every library location because this server resolved no library locations at all.",
+                    skippedUnrootedItems);
+            }
+            else
+            {
+                _logger.LogWarning(
+                    "Keeping {Count} items that are outside every library location because {LocationCount} library location(s) are currently unavailable.",
+                    skippedUnrootedItems,
+                    inaccessiblePaths.Count);
+            }
         }
 
         if (staleIds.Count == 0)
@@ -487,13 +522,50 @@ internal class MigrateLinkedChildren : IDatabaseMigrationRoutine
             return;
         }
 
-        _logger.LogInformation("Found {Count} stale items to remove.", staleIds.Count);
+        // Whether an item went because its own file is gone or because nothing claims its path is the
+        // difference between a handful of deletions and a whole library, so report the split.
+        _logger.LogInformation(
+            "Found {Count} stale items to remove, {UnrootedCount} of them because their path is outside every library location.",
+            staleIds.Count,
+            removedUnrootedItems);
 
         var deleted = ResolveAndDeleteItems(staleIds, "items with missing files");
 
         _logger.LogInformation("Removed {Count} stale items.", deleted);
     }
 
+    // Items under a library whose definition folder is gone. Their locations cannot be resolved, so
+    // they read as unrooted, but the library is what went missing and the media path is still theirs.
+    private HashSet<Guid> GetItemsInMissingLibraries(JellyfinDbContext context, IReadOnlyList<VirtualFolderInfo> virtualFolders)
+    {
+        var liveLibraryPaths = virtualFolders
+            .Select(folder => Path.Combine(_appPaths.DefaultUserViewsPath, folder.Name))
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        var missingLibraryIds = context.BaseItems
+            .Where(b => b.Type == CollectionFolderTypeName && b.Path != null)
+            .Select(b => new { b.Id, b.Path })
+            .ToList()
+            .Where(b => !liveLibraryPaths.Contains(_appHost.ExpandVirtualPath(b.Path!)))
+            .Select(b => b.Id)
+            .ToList();
+
+        if (missingLibraryIds.Count == 0)
+        {
+            return [];
+        }
+
+        _logger.LogWarning(
+            "{Count} library definition(s) known to the database are not in {Path}.",
+            missingLibraryIds.Count,
+            _appPaths.DefaultUserViewsPath);
+
+        return context.AncestorIds
+            .WhereOneOrMany(missingLibraryIds, a => a.ParentItemId)
+            .Select(a => a.ItemId)
+            .ToHashSet();
+    }
+
     private int ResolveAndDeleteItems(IReadOnlyCollection<Guid> ids, string description)
     {
         if (ids.Count == 0)
diff --git a/tests/Jellyfin.Server.Tests/Migrations/MigrateLinkedChildrenStaleItemsTests.cs b/tests/Jellyfin.Server.Tests/Migrations/MigrateLinkedChildrenStaleItemsTests.cs
new file mode 100644
index 0000000000..92fc1dca97
--- /dev/null
+++ b/tests/Jellyfin.Server.Tests/Migrations/MigrateLinkedChildrenStaleItemsTests.cs
@@ -0,0 +1,242 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using Jellyfin.Database.Implementations;
+using Jellyfin.Database.Implementations.Entities;
+using Jellyfin.Database.Implementations.Locking;
+using Jellyfin.Database.Providers.Sqlite;
+using Jellyfin.Server.Migrations.Routines;
+using MediaBrowser.Common.Configuration;
+using MediaBrowser.Controller;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Entities.Audio;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Model.Entities;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging.Abstractions;
+using Moq;
+using Xunit;
+
+namespace Jellyfin.Server.Tests.Migrations;
+
+/// <summary>
+/// Covers which items the stale file cleanup is allowed to delete. Resolving no library locations at
+/// all puts every file-backed item outside every location, so acting on that emptied whole libraries.
+/// </summary>
+public sealed class MigrateLinkedChildrenStaleItemsTests : IDisposable
+{
+    private readonly SqliteConnection _connection;
+    private readonly DbContextOptions<JellyfinDbContext> _dbOptions;
+    private readonly IApplicationPaths _applicationPaths;
+    private readonly string _root;
+
+    public MigrateLinkedChildrenStaleItemsTests()
+    {
+        _applicationPaths = new Mock<IApplicationPaths>().Object;
+        _root = Path.Combine(Path.GetTempPath(), "jf-stale-" + Guid.NewGuid().ToString("N"));
+        Directory.CreateDirectory(Path.Combine(_root, "media"));
+        Directory.CreateDirectory(Path.Combine(_root, "root", "default"));
+
+        _connection = new SqliteConnection("Data Source=:memory:");
+        _connection.Open();
+
+        _dbOptions = new DbContextOptionsBuilder<JellyfinDbContext>()
+            .UseSqlite(_connection)
+            .Options;
+
+        using var context = CreateDbContext();
+        context.Database.EnsureCreated();
+    }
+
+    private string LibraryDefinitionRoot => Path.Combine(_root, "root", "default");
+
+    [Fact]
+    public void Perform_NoLibraryLocationsResolved_KeepsEverything()
+    {
+        var itemId = Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
+        SeedItem(itemId, SeedMediaFile("kept.flac"));
+
+        var deleted = Perform(libraryLocations: []);
+
+        Assert.Empty(deleted);
+    }
+
+    [Fact]
+    public void Perform_ItemOutsideEveryLocation_IsRemoved()
+    {
+        var itemId = Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb");
+
+        // An empty location counts as inaccessible, which disables the removal on purpose, so the
+        // library has to hold something for this case to be reachable at all.
+        SeedMediaFile("anchor.flac");
+        SeedItem(itemId, Path.Combine(_root, "removed-media", "gone.flac"));
+
+        var deleted = Perform(libraryLocations: [Path.Combine(_root, "media")]);
+
+        Assert.Equal([itemId], deleted);
+    }
+
+    [Fact]
+    public void Perform_ItemInsideLocationWithMissingFile_IsRemoved()
+    {
+        var itemId = Guid.Parse("cccccccc-cccc-cccc-cccc-cccccccccccc");
+        SeedMediaFile("anchor.flac");
+        SeedItem(itemId, Path.Combine(_root, "media", "deleted.flac"));
+
+        var deleted = Perform(libraryLocations: [Path.Combine(_root, "media")]);
+
+        Assert.Equal([itemId], deleted);
+    }
+
+    [Fact]
+    public void Perform_ItemInsideLocationWithFilePresent_IsKept()
+    {
+        var itemId = Guid.Parse("dddddddd-dddd-dddd-dddd-dddddddddddd");
+        SeedItem(itemId, SeedMediaFile("present.flac"));
+
+        var deleted = Perform(libraryLocations: [Path.Combine(_root, "media")]);
+
+        Assert.Empty(deleted);
+    }
+
+    [Fact]
+    public void Perform_LibraryDefinitionMissing_KeepsItsItems()
+    {
+        var libraryId = Guid.Parse("11111111-1111-1111-1111-111111111111");
+        var itemId = Guid.Parse("22222222-2222-2222-2222-222222222222");
+
+        // The definition folder of "Movies" is gone, so it contributes no locations and everything it
+        // held reads as unrooted, while "Media" still resolves and makes the run look actionable.
+        SeedLibrary(libraryId, "Movies", createDefinitionFolder: false);
+        SeedMediaFile("anchor.flac");
+        SeedItem(itemId, Path.Combine(_root, "movies", "kept.mkv"), libraryId);
+
+        var deleted = Perform(libraryLocations: [Path.Combine(_root, "media")]);
+
+        Assert.Empty(deleted);
+    }
+
+    [Fact]
+    public void Perform_PathRemovedFromSurvivingLibrary_IsRemoved()
+    {
+        var libraryId = Guid.Parse("33333333-3333-3333-3333-333333333333");
+        var itemId = Guid.Parse("44444444-4444-4444-4444-444444444444");
+
+        // The library is still there; only one of its media paths went away, which is what this
+        // branch exists to clean up.
+        SeedLibrary(libraryId, "Media", createDefinitionFolder: true);
+        SeedMediaFile("anchor.flac");
+        SeedItem(itemId, Path.Combine(_root, "removed-media", "gone.flac"), libraryId);
+
+        var deleted = Perform(libraryLocations: [Path.Combine(_root, "media")]);
+
+        Assert.Equal([itemId], deleted);
+    }
+
+    public void Dispose()
+    {
+        _connection.Dispose();
+        if (Directory.Exists(_root))
+        {
+            Directory.Delete(_root, true);
+        }
+    }
+
+    private JellyfinDbContext CreateDbContext() => new(
+        _dbOptions,
+        NullLogger<JellyfinDbContext>.Instance,
+        new SqliteDatabaseProvider(_applicationPaths, NullLogger<SqliteDatabaseProvider>.Instance),
+        new NoLockBehavior(NullLogger<NoLockBehavior>.Instance));
+
+    private void SeedLibrary(Guid id, string name, bool createDefinitionFolder)
+    {
+        var path = Path.Combine(LibraryDefinitionRoot, name);
+        if (createDefinitionFolder)
+        {
+            Directory.CreateDirectory(path);
+        }
+
+        using var context = CreateDbContext();
+        context.BaseItems.Add(new BaseItemEntity
+        {
+            Id = id,
+            Type = "MediaBrowser.Controller.Entities.CollectionFolder",
+            Name = name,
+            Path = path,
+            IsFolder = true
+        });
+        context.SaveChanges();
+    }
+
+    private string SeedMediaFile(string name)
+    {
+        var path = Path.Combine(_root, "media", name);
+        File.WriteAllBytes(path, [0]);
+        return path;
+    }
+
+    private void SeedItem(Guid id, string path, Guid? libraryId = null)
+    {
+        using var context = CreateDbContext();
+        context.BaseItems.Add(new BaseItemEntity
+        {
+            Id = id,
+            Type = typeof(Audio).FullName!,
+            Name = Path.GetFileNameWithoutExtension(path),
+            Path = path,
+            IsFolder = false,
+            IsVirtualItem = false
+        });
+
+        context.SaveChanges();
+
+        if (libraryId.HasValue)
+        {
+            context.Database.ExecuteSql(
+                $"INSERT INTO AncestorIds (ItemId, ParentItemId) VALUES ({id}, {libraryId.Value})");
+        }
+    }
+
+    private IReadOnlyList<Guid> Perform(string[] libraryLocations, string libraryName = "Media")
+    {
+        var deleted = new List<Guid>();
+
+        var libraryManager = new Mock<ILibraryManager>();
+        libraryManager
+            .Setup(l => l.GetVirtualFolders())
+            .Returns(libraryLocations.Length == 0
+                ? []
+                : [new VirtualFolderInfo { Name = libraryName, Locations = libraryLocations }]);
+        libraryManager
+            .Setup(l => l.GetItemById(It.IsAny<Guid>()))
+            .Returns<Guid>(id => new Audio { Id = id });
+        libraryManager
+            .Setup(l => l.DeleteItem(It.IsAny<BaseItem>(), It.IsAny<DeleteOptions>()))
+            .Callback<BaseItem, DeleteOptions>((item, _) => deleted.Add(item.Id));
+
+        var appHost = new Mock<IServerApplicationHost>();
+        appHost.Setup(h => h.ExpandVirtualPath(It.IsAny<string>())).Returns<string>(p => p);
+
+        var appPaths = new Mock<IServerApplicationPaths>();
+        appPaths.SetupGet(p => p.DataPath).Returns(Path.Combine(_root, "data"));
+        appPaths.SetupGet(p => p.InternalMetadataPath).Returns(Path.Combine(_root, "metadata"));
+        appPaths.SetupGet(p => p.DefaultUserViewsPath).Returns(LibraryDefinitionRoot);
+
+        var factory = new Mock<IDbContextFactory<JellyfinDbContext>>();
+        factory.Setup(f => f.CreateDbContext()).Returns(CreateDbContext);
+        factory.Setup(f => f.CreateDbContextAsync(It.IsAny<CancellationToken>())).ReturnsAsync(CreateDbContext);
+
+        new MigrateLinkedChildren(
+            NullLoggerFactory.Instance,
+            factory.Object,
+            libraryManager.Object,
+            appHost.Object,
+            appPaths.Object)
+            .Perform();
+
+        return deleted;
+    }
+}