diff options
Diffstat (limited to 'Jellyfin.Server.Implementations')
6 files changed, 212 insertions, 87 deletions
diff --git a/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs b/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs index 63c80634f..932f9d625 100644 --- a/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs +++ b/Jellyfin.Server.Implementations/Extensions/ServiceCollectionExtensions.cs @@ -139,7 +139,7 @@ public static class ServiceCollectionExtensions serviceCollection.AddPooledDbContextFactory<JellyfinDbContext>((serviceProvider, opt) => { var provider = serviceProvider.GetRequiredService<IJellyfinDatabaseProvider>(); - provider.Initialise(opt); + provider.Initialise(opt, efCoreConfiguration); var lockingBehavior = serviceProvider.GetRequiredService<IEntityFrameworkCoreLockingBehavior>(); lockingBehavior.Initialise(opt); }); diff --git a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs index e266d5a3b..74d99455d 100644 --- a/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs +++ b/Jellyfin.Server.Implementations/FullSystemBackup/BackupService.cs @@ -39,7 +39,7 @@ public class BackupService : IBackupService ReferenceHandler = ReferenceHandler.IgnoreCycles, }; - private readonly Version _backupEngineVersion = Version.Parse("0.1.0"); + private readonly Version _backupEngineVersion = Version.Parse("0.2.0"); /// <summary> /// Initializes a new instance of the <see cref="BackupService"/> class. @@ -120,26 +120,29 @@ public class BackupService : IBackupService void CopyDirectory(string source, string target) { - source = Path.GetFullPath(source); - Directory.CreateDirectory(source); - + var fullSourcePath = NormalizePathSeparator(Path.GetFullPath(source) + Path.DirectorySeparatorChar); + var fullTargetRoot = Path.GetFullPath(target) + Path.DirectorySeparatorChar; foreach (var item in zipArchive.Entries) { - var sanitizedSourcePath = Path.GetFullPath(item.FullName.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar); - if (!sanitizedSourcePath.StartsWith(target, StringComparison.Ordinal)) + var sourcePath = NormalizePathSeparator(Path.GetFullPath(item.FullName)); + var targetPath = Path.GetFullPath(Path.Combine(target, Path.GetRelativePath(source, item.FullName))); + + if (!sourcePath.StartsWith(fullSourcePath, StringComparison.Ordinal) + || !targetPath.StartsWith(fullTargetRoot, StringComparison.Ordinal)) { continue; } - var targetPath = Path.Combine(source, sanitizedSourcePath[target.Length..].Trim('/')); _logger.LogInformation("Restore and override {File}", targetPath); - item.ExtractToFile(targetPath); + + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + item.ExtractToFile(targetPath, overwrite: true); } } - CopyDirectory(_applicationPaths.ConfigurationDirectoryPath, "Config/"); - CopyDirectory(_applicationPaths.DataPath, "Data/"); - CopyDirectory(_applicationPaths.RootFolderPath, "Root/"); + CopyDirectory("Config", _applicationPaths.ConfigurationDirectoryPath); + CopyDirectory("Data", _applicationPaths.DataPath); + CopyDirectory("Root", _applicationPaths.RootFolderPath); if (manifest.Options.Database) { @@ -148,7 +151,7 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { // restore migration history manually - var historyEntry = zipArchive.GetEntry($"Database\\{nameof(HistoryRow)}.json"); + var historyEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{nameof(HistoryRow)}.json"))); if (historyEntry is null) { _logger.LogInformation("No backup of the history table in archive. This is required for Jellyfin operation"); @@ -165,6 +168,13 @@ public class BackupService : IBackupService var historyRepository = dbContext.GetService<IHistoryRepository>(); await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); + + foreach (var item in await historyRepository.GetAppliedMigrationsAsync(CancellationToken.None).ConfigureAwait(false)) + { + var insertScript = historyRepository.GetDeleteScript(item.MigrationId); + await dbContext.Database.ExecuteSqlRawAsync(insertScript).ConfigureAwait(false); + } + foreach (var item in historyEntries) { var insertScript = historyRepository.GetInsertScript(item); @@ -186,7 +196,7 @@ public class BackupService : IBackupService { _logger.LogInformation("Read backup of {Table}", entityType.Type.Name); - var zipEntry = zipArchive.GetEntry($"Database\\{entityType.Type.Name}.json"); + var zipEntry = zipArchive.GetEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.Type.Name}.json"))); if (zipEntry is null) { _logger.LogInformation("No backup of expected table {Table} is present in backup. Continue anyway.", entityType.Type.Name); @@ -198,7 +208,7 @@ public class BackupService : IBackupService { _logger.LogInformation("Restore backup of {Table}", entityType.Type.Name); var records = 0; - await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable<JsonObject>(zipEntryStream, _serializerSettings).ConfigureAwait(false)!) + await foreach (var item in JsonSerializer.DeserializeAsyncEnumerable<JsonObject>(zipEntryStream, _serializerSettings).ConfigureAwait(false)) { var entity = item.Deserialize(entityType.Type.PropertyType.GetGenericArguments()[0]); if (entity is null) @@ -281,7 +291,7 @@ public class BackupService : IBackupService await using (dbContext.ConfigureAwait(false)) { dbContext.ChangeTracker.QueryTrackingBehavior = QueryTrackingBehavior.NoTracking; - static IAsyncEnumerable<object> GetValues(IQueryable dbSet, Type type) + static IAsyncEnumerable<object> GetValues(IQueryable dbSet) { var method = dbSet.GetType().GetMethod(nameof(DbSet<object>.AsAsyncEnumerable))!; var enumerable = method.Invoke(dbSet, null)!; @@ -292,12 +302,12 @@ public class BackupService : IBackupService var historyRepository = dbContext.GetService<IHistoryRepository>(); var migrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); - ICollection<(Type Type, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes = [ + ICollection<(Type Type, string SourceName, Func<IAsyncEnumerable<object>> ValueFactory)> entityTypes = [ .. typeof(JellyfinDbContext) .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) .Where(e => e.PropertyType.IsAssignableTo(typeof(IQueryable))) - .Select(e => (Type: e.PropertyType, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!, e.PropertyType)))), - (Type: typeof(HistoryRow), ValueFactory: new Func<IAsyncEnumerable<object>>(() => migrations.ToAsyncEnumerable())) + .Select(e => (Type: e.PropertyType, dbContext.Model.FindEntityType(e.PropertyType.GetGenericArguments()[0])!.GetSchemaQualifiedTableName()!, ValueFactory: new Func<IAsyncEnumerable<object>>(() => GetValues((IQueryable)e.GetValue(dbContext)!)))), + (Type: typeof(HistoryRow), SourceName: nameof(HistoryRow), ValueFactory: () => migrations.ToAsyncEnumerable()) ]; manifest.DatabaseTables = entityTypes.Select(e => e.Type.Name).ToArray(); var transaction = await dbContext.Database.BeginTransactionAsync().ConfigureAwait(false); @@ -308,8 +318,8 @@ public class BackupService : IBackupService foreach (var entityType in entityTypes) { - _logger.LogInformation("Begin backup of entity {Table}", entityType.Type.Name); - var zipEntry = zipArchive.CreateEntry($"Database\\{entityType.Type.Name}.json"); + _logger.LogInformation("Begin backup of entity {Table}", entityType.SourceName); + var zipEntry = zipArchive.CreateEntry(NormalizePathSeparator(Path.Combine("Database", $"{entityType.SourceName}.json"))); var entities = 0; var zipEntryStream = zipEntry.Open(); await using (zipEntryStream.ConfigureAwait(false)) @@ -347,7 +357,7 @@ public class BackupService : IBackupService foreach (var item in Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.xml", SearchOption.TopDirectoryOnly) .Union(Directory.EnumerateFiles(_applicationPaths.ConfigurationDirectoryPath, "*.json", SearchOption.TopDirectoryOnly))) { - zipArchive.CreateEntryFromFile(item, Path.Combine("Config", Path.GetFileName(item))); + zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine("Config", Path.GetFileName(item)))); } void CopyDirectory(string source, string target, string filter = "*") @@ -361,7 +371,7 @@ public class BackupService : IBackupService foreach (var item in Directory.EnumerateFiles(source, filter, SearchOption.AllDirectories)) { - zipArchive.CreateEntryFromFile(item, Path.Combine(target, item[..source.Length].Trim('\\'))); + zipArchive.CreateEntryFromFile(item, NormalizePathSeparator(Path.Combine(target, Path.GetRelativePath(source, item)))); } } @@ -509,4 +519,14 @@ public class BackupService : IBackupService Database = options.Database }; } + + /// <summary> + /// Windows is able to handle '/' as a path seperator in zip files + /// but linux isn't able to handle '\' as a path seperator in zip files, + /// So normalize to '/'. + /// </summary> + /// <param name="path">The path to normalize.</param> + /// <returns>The normalized path. </returns> + private static string NormalizePathSeparator(string path) + => path.Replace('\\', '/'); } diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs index 3efcb6dd3..d59eba690 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs @@ -14,6 +14,7 @@ using System.Reflection; using System.Text; using System.Text.Json; using System.Threading; +using System.Threading.Tasks; using Jellyfin.Data.Enums; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.Entities; @@ -54,6 +55,11 @@ public sealed class BaseItemRepository : IItemRepository { /// <summary> + /// Gets the placeholder id for UserData detached items. + /// </summary> + public static readonly Guid PlaceholderId = Guid.Parse("00000000-0000-0000-0000-000000000001"); + + /// <summary> /// This holds all the types in the running assemblies /// so that we can de-serialize properly when we don't have strong types. /// </summary> @@ -95,13 +101,35 @@ public sealed class BaseItemRepository /// <inheritdoc /> public void DeleteItem(Guid id) { - if (id.IsEmpty()) + if (id.IsEmpty() || id.Equals(PlaceholderId)) { - throw new ArgumentException("Guid can't be empty", nameof(id)); + throw new ArgumentException("Guid can't be empty or the placeholder id.", nameof(id)); } using var context = _dbProvider.CreateDbContext(); using var transaction = context.Database.BeginTransaction(); + + var date = (DateTime?)DateTime.UtcNow; + + // Remove any UserData entries for the placeholder item that would conflict with the UserData + // being detached from the item being deleted. This is necessary because, during an update, + // UserData may be reattached to a new entry, but some entries can be left behind. + // Ensures there are no duplicate UserId/CustomDataKey combinations for the placeholder. + context.UserData + .Join( + context.UserData.Where(e => e.ItemId == id), + placeholder => new { placeholder.UserId, placeholder.CustomDataKey }, + userData => new { userData.UserId, userData.CustomDataKey }, + (placeholder, userData) => placeholder) + .Where(e => e.ItemId == PlaceholderId) + .ExecuteDelete(); + + // Detach all user watch data + context.UserData.Where(e => e.ItemId == id) + .ExecuteUpdate(e => e + .SetProperty(f => f.RetentionDate, date) + .SetProperty(f => f.ItemId, PlaceholderId)); + context.AncestorIds.Where(e => e.ItemId == id || e.ParentItemId == id).ExecuteDelete(); context.AttachmentStreamInfos.Where(e => e.ItemId == id).ExecuteDelete(); context.BaseItemImageInfos.Where(e => e.ItemId == id).ExecuteDelete(); @@ -144,7 +172,7 @@ public sealed class BaseItemRepository PrepareFilterQuery(filter); using var context = _dbProvider.CreateDbContext(); - return ApplyQueryFilter(context.BaseItems.AsNoTracking(), context, filter).Select(e => e.Id).ToArray(); + return ApplyQueryFilter(context.BaseItems.AsNoTracking().Where(e => e.Id != EF.Constant(PlaceholderId)), context, filter).Select(e => e.Id).ToArray(); } /// <inheritdoc /> @@ -242,7 +270,7 @@ public sealed class BaseItemRepository dbQuery = ApplyGroupingFilter(dbQuery, filter); dbQuery = ApplyQueryPaging(dbQuery, filter); - result.Items = dbQuery.AsEnumerable().Where(e => e is not null).Select(w => DeserialiseBaseItem(w, filter.SkipDeserialization)).ToArray(); + result.Items = dbQuery.AsEnumerable().Where(e => e is not null).Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)).ToArray(); result.StartIndex = filter.StartIndex ?? 0; return result; } @@ -261,7 +289,7 @@ public sealed class BaseItemRepository dbQuery = ApplyGroupingFilter(dbQuery, filter); dbQuery = ApplyQueryPaging(dbQuery, filter); - return dbQuery.AsEnumerable().Where(e => e is not null).Select(w => DeserialiseBaseItem(w, filter.SkipDeserialization)).ToArray(); + return dbQuery.AsEnumerable().Where(e => e is not null).Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)).ToArray(); } /// <inheritdoc/> @@ -303,7 +331,7 @@ public sealed class BaseItemRepository mainquery = ApplyGroupingFilter(mainquery, filter); mainquery = ApplyQueryPaging(mainquery, filter); - return mainquery.AsEnumerable().Where(e => e is not null).Select(w => DeserialiseBaseItem(w, filter.SkipDeserialization)).ToArray(); + return mainquery.AsEnumerable().Where(e => e is not null).Select(w => DeserializeBaseItem(w, filter.SkipDeserialization)).ToArray(); } /// <inheritdoc /> @@ -319,7 +347,7 @@ public sealed class BaseItemRepository .Where(i => filter.TopParentIds.Contains(i.TopParentId!.Value)) .Where(i => i.Type == _itemTypeLookup.BaseItemKindNames[BaseItemKind.Episode]) .Join( - context.UserData.AsNoTracking(), + context.UserData.AsNoTracking().Where(e => e.ItemId != EF.Constant(PlaceholderId)), i => new { UserId = filter.User.Id, ItemId = i.Id }, u => new { UserId = u.UserId, ItemId = u.ItemId }, (entity, data) => new { Item = entity, UserData = data }) @@ -454,6 +482,13 @@ public sealed class BaseItemRepository var images = item.ImageInfos.Select(e => Map(item.Id, e)); using var context = _dbProvider.CreateDbContext(); + + if (!context.BaseItems.Any(bi => bi.Id == item.Id)) + { + _logger.LogWarning("Unable to save ImageInfo for non existing BaseItem"); + return; + } + context.BaseItemImageInfos.Where(e => e.ItemId == item.Id).ExecuteDelete(); context.BaseItemImageInfos.AddRange(images); context.SaveChanges(); @@ -472,7 +507,7 @@ public sealed class BaseItemRepository cancellationToken.ThrowIfCancellationRequested(); var tuples = new List<(BaseItemDto Item, List<Guid>? AncestorIds, BaseItemDto TopParent, IEnumerable<string> UserDataKey, List<string> InheritedTags)>(); - foreach (var item in items.GroupBy(e => e.Id).Select(e => e.Last())) + foreach (var item in items.GroupBy(e => e.Id).Select(e => e.Last()).Where(e => e.Id != PlaceholderId)) { var ancestorIds = item.SupportsAncestors ? item.GetAncestorIds().Distinct().ToList() : @@ -491,6 +526,7 @@ public sealed class BaseItemRepository var ids = tuples.Select(f => f.Item.Id).ToArray(); var existingItems = context.BaseItems.Where(e => ids.Contains(e.Id)).Select(f => f.Id).ToArray(); + var newItems = tuples.Where(e => !existingItems.Contains(e.Item.Id)).ToArray(); foreach (var item in tuples) { @@ -511,8 +547,21 @@ public sealed class BaseItemRepository context.SaveChanges(); + foreach (var item in newItems) + { + // reattach old userData entries + var userKeys = item.UserDataKey.ToArray(); + var retentionDate = (DateTime?)null; + context.UserData + .Where(e => e.ItemId == PlaceholderId) + .Where(e => userKeys.Contains(e.CustomDataKey)) + .ExecuteUpdate(e => e + .SetProperty(f => f.ItemId, item.Item.Id) + .SetProperty(f => f.RetentionDate, retentionDate)); + } + var itemValueMaps = tuples - .Select(e => (Item: e.Item, Values: GetItemValuesToSave(e.Item, e.InheritedTags))) + .Select(e => (e.Item, Values: GetItemValuesToSave(e.Item, e.InheritedTags))) .ToArray(); var allListedItemValues = itemValueMaps .SelectMany(f => f.Values) @@ -539,7 +588,7 @@ public sealed class BaseItemRepository var itemValuesStore = existingValues.Concat(missingItemValues).ToArray(); var valueMap = itemValueMaps - .Select(f => (Item: f.Item, Values: f.Values.Select(e => itemValuesStore.First(g => g.Value == e.Value && g.Type == e.MagicNumber)).ToArray())) + .Select(f => (f.Item, Values: f.Values.Select(e => itemValuesStore.First(g => g.Value == e.Value && g.Type == e.MagicNumber)).DistinctBy(e => e.ItemValueId).ToArray())) .ToArray(); var mappedValues = context.ItemValuesMap.Where(e => ids.Contains(e.ItemId)).ToList(); @@ -627,7 +676,7 @@ public sealed class BaseItemRepository return null; } - return DeserialiseBaseItem(item); + return DeserializeBaseItem(item); } /// <summary> @@ -673,12 +722,12 @@ public sealed class BaseItemRepository dto.TotalBitrate = entity.TotalBitrate; dto.ExternalId = entity.ExternalId; dto.Size = entity.Size; - dto.Genres = entity.Genres?.Split('|') ?? []; - dto.DateCreated = entity.DateCreated.GetValueOrDefault(); - dto.DateModified = entity.DateModified.GetValueOrDefault(); + dto.Genres = string.IsNullOrWhiteSpace(entity.Genres) ? [] : entity.Genres.Split('|'); + dto.DateCreated = entity.DateCreated ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); + dto.DateModified = entity.DateModified ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); dto.ChannelId = entity.ChannelId ?? Guid.Empty; - dto.DateLastRefreshed = entity.DateLastRefreshed.GetValueOrDefault(); - dto.DateLastSaved = entity.DateLastSaved.GetValueOrDefault(); + dto.DateLastRefreshed = entity.DateLastRefreshed ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); + dto.DateLastSaved = entity.DateLastSaved ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); dto.OwnerId = string.IsNullOrWhiteSpace(entity.OwnerId) ? Guid.Empty : (Guid.TryParse(entity.OwnerId, out var ownerId) ? ownerId : Guid.Empty); dto.Width = entity.Width.GetValueOrDefault(); dto.Height = entity.Height.GetValueOrDefault(); @@ -705,7 +754,7 @@ public sealed class BaseItemRepository dto.ExtraIds = string.IsNullOrWhiteSpace(entity.ExtraIds) ? [] : entity.ExtraIds.Split('|').Select(e => Guid.Parse(e)).ToArray(); dto.ProductionLocations = entity.ProductionLocations?.Split('|') ?? []; dto.Studios = entity.Studios?.Split('|') ?? []; - dto.Tags = entity.Tags?.Split('|') ?? []; + dto.Tags = string.IsNullOrWhiteSpace(entity.Tags) ? [] : entity.Tags.Split('|'); if (dto is IHasProgramAttributes hasProgramAttributes) { @@ -779,7 +828,7 @@ public sealed class BaseItemRepository if (dto is Folder folder) { - folder.DateLastMediaAdded = entity.DateLastMediaAdded; + folder.DateLastMediaAdded = entity.DateLastMediaAdded ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc); } return dto; @@ -839,11 +888,11 @@ public sealed class BaseItemRepository entity.ExternalId = dto.ExternalId; entity.Size = dto.Size; entity.Genres = string.Join('|', dto.Genres); - entity.DateCreated = dto.DateCreated; - entity.DateModified = dto.DateModified; + entity.DateCreated = dto.DateCreated == DateTime.MinValue ? null : dto.DateCreated; + entity.DateModified = dto.DateModified == DateTime.MinValue ? null : dto.DateModified; entity.ChannelId = dto.ChannelId; - entity.DateLastRefreshed = dto.DateLastRefreshed; - entity.DateLastSaved = dto.DateLastSaved; + entity.DateLastRefreshed = dto.DateLastRefreshed == DateTime.MinValue ? null : dto.DateLastRefreshed; + entity.DateLastSaved = dto.DateLastSaved == DateTime.MinValue ? null : dto.DateLastSaved; entity.OwnerId = dto.OwnerId.ToString(); entity.Width = dto.Width; entity.Height = dto.Height; @@ -953,7 +1002,7 @@ public sealed class BaseItemRepository if (dto is Folder folder) { - entity.DateLastMediaAdded = folder.DateLastMediaAdded; + entity.DateLastMediaAdded = folder.DateLastMediaAdded == DateTime.MinValue ? null : folder.DateLastMediaAdded; entity.IsFolder = folder.IsFolder; } @@ -989,7 +1038,7 @@ public sealed class BaseItemRepository return type.GetCustomAttribute<RequiresSourceSerialisationAttribute>() == null; } - private BaseItemDto DeserialiseBaseItem(BaseItemEntity baseItemEntity, bool skipDeserialization = false) + private BaseItemDto DeserializeBaseItem(BaseItemEntity baseItemEntity, bool skipDeserialization = false) { ArgumentNullException.ThrowIfNull(baseItemEntity, nameof(baseItemEntity)); if (_serverConfigurationManager?.Configuration is null) @@ -998,7 +1047,7 @@ public sealed class BaseItemRepository } var typeToSerialise = GetType(baseItemEntity.Type); - return BaseItemRepository.DeserialiseBaseItem( + return BaseItemRepository.DeserializeBaseItem( baseItemEntity, _logger, _appHost, @@ -1006,7 +1055,7 @@ public sealed class BaseItemRepository } /// <summary> - /// Deserialises a BaseItemEntity and sets all properties. + /// Deserializes a BaseItemEntity and sets all properties. /// </summary> /// <param name="baseItemEntity">The DB entity.</param> /// <param name="logger">Logger.</param> @@ -1014,9 +1063,9 @@ public sealed class BaseItemRepository /// <param name="skipDeserialization">If only mapping should be processed.</param> /// <returns>A mapped BaseItem.</returns> /// <exception cref="InvalidOperationException">Will be thrown if an invalid serialisation is requested.</exception> - public static BaseItemDto DeserialiseBaseItem(BaseItemEntity baseItemEntity, ILogger logger, IServerApplicationHost? appHost, bool skipDeserialization = false) + public static BaseItemDto DeserializeBaseItem(BaseItemEntity baseItemEntity, ILogger logger, IServerApplicationHost? appHost, bool skipDeserialization = false) { - var type = GetType(baseItemEntity.Type) ?? throw new InvalidOperationException("Cannot deserialise unknown type."); + var type = GetType(baseItemEntity.Type) ?? throw new InvalidOperationException("Cannot deserialize unknown type."); BaseItemDto? dto = null; if (TypeRequiresDeserialization(type) && baseItemEntity.Data is not null && !skipDeserialization) { @@ -1032,7 +1081,7 @@ public sealed class BaseItemRepository if (dto is null) { - dto = Activator.CreateInstance(type) as BaseItemDto ?? throw new InvalidOperationException("Cannot deserialise unknown type."); + dto = Activator.CreateInstance(type) as BaseItemDto ?? throw new InvalidOperationException("Cannot deserialize unknown type."); } return Map(baseItemEntity, dto, appHost); @@ -1049,7 +1098,7 @@ public sealed class BaseItemRepository using var context = _dbProvider.CreateDbContext(); - var innerQueryFilter = TranslateQuery(context.BaseItems, context, new InternalItemsQuery(filter.User) + var innerQueryFilter = TranslateQuery(context.BaseItems.Where(e => e.Id != EF.Constant(PlaceholderId)), context, new InternalItemsQuery(filter.User) { ExcludeItemTypes = filter.ExcludeItemTypes, IncludeItemTypes = filter.IncludeItemTypes, @@ -1138,7 +1187,7 @@ public sealed class BaseItemRepository IsPlayed = filter.IsPlayed }; - itemCountQuery = TranslateQuery(context.BaseItems.AsNoTracking(), context, typeSubQuery) + itemCountQuery = TranslateQuery(context.BaseItems.AsNoTracking().Where(e => e.Id != EF.Constant(PlaceholderId)), context, typeSubQuery) .Where(e => e.ItemValues!.Any(f => itemValueTypes!.Contains(f.ItemValue.Type))); var seriesTypeName = _itemTypeLookup.BaseItemKindNames[BaseItemKind.Series]; @@ -1178,7 +1227,7 @@ public sealed class BaseItemRepository .Where(e => e is not null) .Select(e => { - return (DeserialiseBaseItem(e.item, filter.SkipDeserialization), e.itemCount); + return (DeserializeBaseItem(e.item, filter.SkipDeserialization), e.itemCount); }) ]; } @@ -1193,7 +1242,7 @@ public sealed class BaseItemRepository .Where(e => e is not null) .Select<BaseItemEntity, (BaseItemDto, ItemCounts?)>(e => { - return (DeserialiseBaseItem(e, filter.SkipDeserialization), null); + return (DeserializeBaseItem(e, filter.SkipDeserialization), null); }) ]; } @@ -1274,7 +1323,7 @@ public sealed class BaseItemRepository { Path = appHost?.ExpandVirtualPath(e.Path) ?? e.Path, BlurHash = e.Blurhash is null ? null : Encoding.UTF8.GetString(e.Blurhash), - DateModified = e.DateModified, + DateModified = e.DateModified ?? DateTime.SpecifyKind(DateTime.MinValue, DateTimeKind.Utc), Height = e.Height, Width = e.Width, Type = (ImageType)e.ImageType @@ -1814,7 +1863,7 @@ public sealed class BaseItemRepository // We should probably figure this out for all folders, but for right now, this is the only place where we need it if (filter.IncludeItemTypes.Length == 1 && filter.IncludeItemTypes[0] == BaseItemKind.Series) { - baseQuery = baseQuery.Where(e => context.BaseItems + baseQuery = baseQuery.Where(e => context.BaseItems.Where(e => e.Id != EF.Constant(PlaceholderId)) .Where(e => e.IsFolder == false && e.IsVirtualItem == false) .Where(f => f.UserData!.FirstOrDefault(e => e.UserId == filter.User!.Id && e.Played)!.Played) .Any(f => f.SeriesPresentationUniqueKey == e.PresentationUniqueKey) == filter.IsPlayed); @@ -2064,7 +2113,7 @@ public sealed class BaseItemRepository if (filter.HasDeadParentId.HasValue && filter.HasDeadParentId.Value) { baseQuery = baseQuery - .Where(e => e.ParentId.HasValue && !context.BaseItems.Any(f => f.Id == e.ParentId.Value)); + .Where(e => e.ParentId.HasValue && !context.BaseItems.Where(e => e.Id != EF.Constant(PlaceholderId)).Any(f => f.Id == e.ParentId.Value)); } if (filter.IsDeadArtist.HasValue && filter.IsDeadArtist.Value) @@ -2145,17 +2194,19 @@ public sealed class BaseItemRepository if (filter.ExcludeItemIds.Length > 0) { baseQuery = baseQuery - .Where(e => !filter.ItemIds.Contains(e.Id)); + .Where(e => !filter.ExcludeItemIds.Contains(e.Id)); } if (filter.ExcludeProviderIds is not null && filter.ExcludeProviderIds.Count > 0) { - baseQuery = baseQuery.Where(e => !e.Provider!.All(f => !filter.ExcludeProviderIds.All(w => f.ProviderId == w.Key && f.ProviderValue == w.Value))); + var exclude = filter.ExcludeProviderIds.Select(e => $"{e.Key}:{e.Value}").ToArray(); + baseQuery = baseQuery.Where(e => e.Provider!.Select(f => f.ProviderId + ":" + f.ProviderValue)!.All(f => !exclude.Contains(f))); } if (filter.HasAnyProviderId is not null && filter.HasAnyProviderId.Count > 0) { - baseQuery = baseQuery.Where(e => e.Provider!.Any(f => !filter.HasAnyProviderId.Any(w => f.ProviderId == w.Key && f.ProviderValue == w.Value))); + var include = filter.HasAnyProviderId.Select(e => $"{e.Key}:{e.Value}").ToArray(); + baseQuery = baseQuery.Where(e => e.Provider!.Select(f => f.ProviderId + ":" + f.ProviderValue)!.Any(f => include.Contains(f))); } if (filter.HasImdbId.HasValue) @@ -2197,7 +2248,7 @@ public sealed class BaseItemRepository if (!string.IsNullOrWhiteSpace(filter.AncestorWithPresentationUniqueKey)) { baseQuery = baseQuery - .Where(e => context.BaseItems.Where(f => f.PresentationUniqueKey == filter.AncestorWithPresentationUniqueKey).Any(f => f.Children!.Any(w => w.ItemId == e.Id))); + .Where(e => context.BaseItems.Where(e => e.Id != EF.Constant(PlaceholderId)).Where(f => f.PresentationUniqueKey == filter.AncestorWithPresentationUniqueKey).Any(f => f.Children!.Any(w => w.ItemId == e.Id))); } if (!string.IsNullOrWhiteSpace(filter.SeriesPresentationUniqueKey)) @@ -2209,8 +2260,8 @@ public sealed class BaseItemRepository if (filter.ExcludeInheritedTags.Length > 0) { baseQuery = baseQuery - .Where(e => !e.ItemValues!.Where(w => w.ItemValue.Type == ItemValueType.InheritedTags) - .Any(f => filter.ExcludeInheritedTags.Contains(f.ItemValue.CleanValue))); + .Where(e => !e.ItemValues!.Where(w => w.ItemValue.Type == ItemValueType.InheritedTags || w.ItemValue.Type == ItemValueType.Tags) + .Any(f => filter.ExcludeInheritedTags.Contains(f.ItemValue.CleanValue))); } if (filter.IncludeInheritedTags.Length > 0) @@ -2220,10 +2271,10 @@ public sealed class BaseItemRepository if (includeTypes.Length == 1 && includeTypes.FirstOrDefault() is BaseItemKind.Episode) { baseQuery = baseQuery - .Where(e => e.ItemValues!.Where(f => f.ItemValue.Type == ItemValueType.InheritedTags) + .Where(e => e.ItemValues!.Where(f => f.ItemValue.Type == ItemValueType.InheritedTags || f.ItemValue.Type == ItemValueType.Tags) .Any(f => filter.IncludeInheritedTags.Contains(f.ItemValue.CleanValue)) || - (e.ParentId.HasValue && context.ItemValuesMap.Where(w => w.ItemId == e.ParentId.Value)!.Where(w => w.ItemValue.Type == ItemValueType.InheritedTags) + (e.ParentId.HasValue && context.ItemValuesMap.Where(w => w.ItemId == e.ParentId.Value && (w.ItemValue.Type == ItemValueType.InheritedTags || w.ItemValue.Type == ItemValueType.Tags)) .Any(f => filter.IncludeInheritedTags.Contains(f.ItemValue.CleanValue)))); } @@ -2231,17 +2282,16 @@ public sealed class BaseItemRepository else if (includeTypes.Length == 1 && includeTypes.FirstOrDefault() is BaseItemKind.Playlist) { baseQuery = baseQuery - .Where(e => - e.Parents! - .Any(f => - f.ParentItem.ItemValues!.Any(w => w.ItemValue.Type == ItemValueType.Tags && filter.IncludeInheritedTags.Contains(w.ItemValue.CleanValue)) - || e.Data!.Contains($"OwnerUserId\":\"{filter.User!.Id:N}\""))); + .Where(e => e.ItemValues!.Where(f => f.ItemValue.Type == ItemValueType.InheritedTags || f.ItemValue.Type == ItemValueType.Tags) + .Any(f => filter.IncludeInheritedTags.Contains(f.ItemValue.CleanValue)) + || e.Data!.Contains($"OwnerUserId\":\"{filter.User!.Id:N}\"")); // d ^^ this is stupid it hate this. } else { baseQuery = baseQuery - .Where(e => e.Parents!.Any(f => f.ParentItem.ItemValues!.Any(w => w.ItemValue.Type == ItemValueType.Tags && filter.IncludeInheritedTags.Contains(w.ItemValue.CleanValue)))); + .Where(e => e.ItemValues!.Where(f => f.ItemValue.Type == ItemValueType.InheritedTags || f.ItemValue.Type == ItemValueType.Tags) + .Any(f => filter.IncludeInheritedTags.Contains(f.ItemValue.CleanValue))); } } @@ -2324,4 +2374,14 @@ public sealed class BaseItemRepository return baseQuery; } + + /// <inheritdoc/> + public async Task<bool> ItemExistsAsync(Guid id) + { + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + return await dbContext.BaseItems.AnyAsync(f => f.Id == id).ConfigureAwait(false); + } + } } diff --git a/Jellyfin.Server.Implementations/Item/MediaAttachmentRepository.cs b/Jellyfin.Server.Implementations/Item/MediaAttachmentRepository.cs index 3ae6dbd70..e75dda439 100644 --- a/Jellyfin.Server.Implementations/Item/MediaAttachmentRepository.cs +++ b/Jellyfin.Server.Implementations/Item/MediaAttachmentRepository.cs @@ -25,8 +25,16 @@ public class MediaAttachmentRepository(IDbContextFactory<JellyfinDbContext> dbPr { using var context = dbProvider.CreateDbContext(); using var transaction = context.Database.BeginTransaction(); + + // Users may replace a media with a version that includes attachments to one without them. + // So when saving attachments is triggered by a library scan, we always unconditionally + // clear the old ones, and then add the new ones if given. context.AttachmentStreamInfos.Where(e => e.ItemId.Equals(id)).ExecuteDelete(); - context.AttachmentStreamInfos.AddRange(attachments.Select(e => Map(e, id))); + if (attachments.Any()) + { + context.AttachmentStreamInfos.AddRange(attachments.Select(e => Map(e, id))); + } + context.SaveChanges(); transaction.Commit(); } diff --git a/Jellyfin.Server.Implementations/MediaSegments/MediaSegmentManager.cs b/Jellyfin.Server.Implementations/MediaSegments/MediaSegmentManager.cs index 28b6890b0..97c9d79f5 100644 --- a/Jellyfin.Server.Implementations/MediaSegments/MediaSegmentManager.cs +++ b/Jellyfin.Server.Implementations/MediaSegments/MediaSegmentManager.cs @@ -51,7 +51,7 @@ public class MediaSegmentManager : IMediaSegmentManager } /// <inheritdoc/> - public async Task RunSegmentPluginProviders(BaseItem baseItem, LibraryOptions libraryOptions, bool overwrite, CancellationToken cancellationToken) + public async Task RunSegmentPluginProviders(BaseItem baseItem, LibraryOptions libraryOptions, bool forceOverwrite, CancellationToken cancellationToken) { var providers = _segmentProviders .Where(e => !libraryOptions.DisabledMediaSegmentProviders.Contains(GetProviderId(e.Name))) @@ -70,18 +70,13 @@ public class MediaSegmentManager : IMediaSegmentManager using var db = await _dbProvider.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); - if (!overwrite && (await db.MediaSegments.AnyAsync(e => e.ItemId.Equals(baseItem.Id), cancellationToken).ConfigureAwait(false))) - { - _logger.LogDebug("Skip {MediaPath} as it already contains media segments", baseItem.Path); - return; - } - _logger.LogDebug("Start media segment extraction for {MediaPath} with {CountProviders} providers enabled", baseItem.Path, providers.Count); - await db.MediaSegments.Where(e => e.ItemId.Equals(baseItem.Id)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); - - // no need to recreate the request object every time. - var requestItem = new MediaSegmentGenerationRequest() { ItemId = baseItem.Id }; + if (forceOverwrite) + { + // delete all existing media segments if forceOverwrite is set. + await db.MediaSegments.Where(e => e.ItemId.Equals(baseItem.Id)).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + } foreach (var provider in providers) { @@ -91,15 +86,56 @@ public class MediaSegmentManager : IMediaSegmentManager continue; } + IQueryable<MediaSegment> existingSegments; + if (forceOverwrite) + { + existingSegments = Array.Empty<MediaSegment>().AsQueryable(); + } + else + { + existingSegments = db.MediaSegments.Where(e => e.ItemId.Equals(baseItem.Id) && e.SegmentProviderId == GetProviderId(provider.Name)); + } + + var requestItem = new MediaSegmentGenerationRequest() + { + ItemId = baseItem.Id, + ExistingSegments = existingSegments.Select(e => Map(e)).ToArray() + }; + try { var segments = await provider.GetMediaSegments(requestItem, cancellationToken) .ConfigureAwait(false); - if (segments.Count == 0) + + if (!forceOverwrite) + { + var existingSegmentsList = existingSegments.ToArray(); // Cannot use requestItem's list, as the provider might tamper with its items. + if (segments.Count == requestItem.ExistingSegments.Count && segments.All(e => existingSegmentsList.Any(f => + { + return + e.StartTicks == f.StartTicks && + e.EndTicks == f.EndTicks && + e.Type == f.Type; + }))) + { + _logger.LogDebug("Media Segment provider {ProviderName} did not modify any segments for {MediaPath}", provider.Name, baseItem.Path); + continue; + } + + // delete existing media segments that were re-generated. + await existingSegments.ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + } + + if (segments.Count == 0 && !requestItem.ExistingSegments.Any()) { _logger.LogDebug("Media Segment provider {ProviderName} did not find any segments for {MediaPath}", provider.Name, baseItem.Path); continue; } + else if (segments.Count == 0 && requestItem.ExistingSegments.Any()) + { + _logger.LogDebug("Media Segment provider {ProviderName} deleted all segments for {MediaPath}", provider.Name, baseItem.Path); + continue; + } _logger.LogInformation("Media Segment provider {ProviderName} found {CountSegments} for {MediaPath}", provider.Name, segments.Count, baseItem.Path); var providerId = GetProviderId(provider.Name); diff --git a/Jellyfin.Server.Implementations/Users/UserManager.cs b/Jellyfin.Server.Implementations/Users/UserManager.cs index 3dfb14d71..4f944c87d 100644 --- a/Jellyfin.Server.Implementations/Users/UserManager.cs +++ b/Jellyfin.Server.Implementations/Users/UserManager.cs @@ -744,7 +744,8 @@ namespace Jellyfin.Server.Implementations.Users _users[user.Id] = user; } - internal static void ThrowIfInvalidUsername(string name) + /// <inheritdoc/> + public void ThrowIfInvalidUsername(string name) { if (!string.IsNullOrWhiteSpace(name) && ValidUsernameRegex().IsMatch(name)) { |