diff options
Diffstat (limited to 'src/Jellyfin.Database/Jellyfin.Database.Implementations')
24 files changed, 1180 insertions, 20 deletions
diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs new file mode 100644 index 000000000..fcb8f41b3 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOption.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// The custom value option for custom database providers. +/// </summary> +public class CustomDatabaseOption +{ + /// <summary> + /// Gets or sets the key of the value. + /// </summary> + public required string Key { get; set; } + + /// <summary> + /// Gets or sets the value. + /// </summary> + public required string Value { get; set; } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs new file mode 100644 index 000000000..e2088704d --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/CustomDatabaseOptions.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// Defines the options for a custom database connector. +/// </summary> +public class CustomDatabaseOptions +{ + /// <summary> + /// Gets or sets the Plugin name to search for database providers. + /// </summary> + public required string PluginName { get; set; } + + /// <summary> + /// Gets or sets the plugin assembly to search for providers. + /// </summary> + public required string PluginAssembly { get; set; } + + /// <summary> + /// Gets or sets the connection string for the custom database provider. + /// </summary> + public required string ConnectionString { get; set; } + + /// <summary> + /// Gets or sets the list of extra options for the custom provider. + /// </summary> +#pragma warning disable CA2227 // Collection properties should be read only + public Collection<CustomDatabaseOption> Options { get; set; } = []; +#pragma warning restore CA2227 // Collection properties should be read only +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs index b481a106f..bc0cacf3c 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseConfigurationOptions.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; + namespace Jellyfin.Database.Implementations.DbConfiguration; /// <summary> @@ -9,4 +11,15 @@ public class DatabaseConfigurationOptions /// Gets or Sets the type of database jellyfin should use. /// </summary> public required string DatabaseType { get; set; } + + /// <summary> + /// Gets or sets the options required to use a custom database provider. + /// </summary> + public CustomDatabaseOptions? CustomProviderOptions { get; set; } + + /// <summary> + /// Gets or Sets the kind of locking behavior jellyfin should perform. Possible options are "NoLock", "Pessimistic", "Optimistic". + /// Defaults to "NoLock". + /// </summary> + public DatabaseLockingBehaviorTypes LockingBehavior { get; set; } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs new file mode 100644 index 000000000..3b2a55802 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/DbConfiguration/DatabaseLockingBehaviorTypes.cs @@ -0,0 +1,22 @@ +namespace Jellyfin.Database.Implementations.DbConfiguration; + +/// <summary> +/// Defines all possible methods for locking database access for concurrent queries. +/// </summary> +public enum DatabaseLockingBehaviorTypes +{ + /// <summary> + /// Defines that no explicit application level locking for reads and writes should be done and only provider specific locking should be relied on. + /// </summary> + NoLock = 0, + + /// <summary> + /// Defines a behavior that always blocks all reads while any one write is done. + /// </summary> + Pessimistic = 1, + + /// <summary> + /// Defines that all writes should be attempted and when fail should be retried. + /// </summary> + Optimistic = 2 +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/AttachmentStreamInfo.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/AttachmentStreamInfo.cs index aab3082b3..2f27d9389 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/AttachmentStreamInfo.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/AttachmentStreamInfo.cs @@ -25,7 +25,7 @@ public class AttachmentStreamInfo /// <summary> /// Gets or Sets the codec of the attachment. /// </summary> - public required string Codec { get; set; } + public string? Codec { get; set; } /// <summary> /// Gets or Sets the codec tag of the attachment. diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/BaseItemEntity.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/BaseItemEntity.cs index fc9695a09..a09a96317 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/BaseItemEntity.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/BaseItemEntity.cs @@ -84,6 +84,8 @@ public class BaseItemEntity public int? InheritedParentalRatingValue { get; set; } + public int? InheritedParentalRatingSubValue { get; set; } + public string? UnratedType { get; set; } public float? CriticRating { get; set; } @@ -162,7 +164,7 @@ public class BaseItemEntity public ICollection<BaseItemProvider>? Provider { get; set; } - public ICollection<AncestorId>? ParentAncestors { get; set; } + public ICollection<AncestorId>? Parents { get; set; } public ICollection<AncestorId>? Children { get; set; } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/KeyframeData.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/KeyframeData.cs new file mode 100644 index 000000000..c34110c4f --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/KeyframeData.cs @@ -0,0 +1,32 @@ +#pragma warning disable CA2227 // Collection properties should be read only + +using System; +using System.Collections.Generic; + +namespace Jellyfin.Database.Implementations.Entities; + +/// <summary> +/// Keyframe information for a specific file. +/// </summary> +public class KeyframeData +{ + /// <summary> + /// Gets or Sets the ItemId. + /// </summary> + public required Guid ItemId { get; set; } + + /// <summary> + /// Gets or sets the total duration of the stream in ticks. + /// </summary> + public long TotalDuration { get; set; } + + /// <summary> + /// Gets or sets the keyframes in ticks. + /// </summary> + public ICollection<long>? KeyframeTicks { get; set; } + + /// <summary> + /// Gets or sets the item reference. + /// </summary> + public BaseItemEntity? Item { get; set; } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/MediaStreamInfo.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/MediaStreamInfo.cs index 207317376..b80b764ba 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/MediaStreamInfo.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/MediaStreamInfo.cs @@ -99,4 +99,6 @@ public class MediaStreamInfo public int? Rotation { get; set; } public string? KeyFrames { get; set; } + + public bool? Hdr10PlusPresentFlag { get; set; } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/TrickplayInfo.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/TrickplayInfo.cs index 06b290e4f..39b449553 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/TrickplayInfo.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/TrickplayInfo.cs @@ -14,7 +14,6 @@ public class TrickplayInfo /// <remarks> /// Required. /// </remarks> - [JsonIgnore] public Guid ItemId { get; set; } /// <summary> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/User.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/User.cs index 31538b5bf..6c81fa729 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/User.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Entities/User.cs @@ -61,7 +61,6 @@ namespace Jellyfin.Database.Implementations.Entities /// <remarks> /// Identity, Indexed, Required. /// </remarks> - [JsonIgnore] public Guid Id { get; set; } /// <summary> @@ -249,9 +248,14 @@ namespace Jellyfin.Database.Implementations.Entities public bool EnableUserPreferenceAccess { get; set; } /// <summary> - /// Gets or sets the maximum parental age rating. + /// Gets or sets the maximum parental rating score. /// </summary> - public int? MaxParentalAgeRating { get; set; } + public int? MaxParentalRatingScore { get; set; } + + /// <summary> + /// Gets or sets the maximum parental rating sub score. + /// </summary> + public int? MaxParentalRatingSubScore { get; set; } /// <summary> /// Gets or sets the remote client bitrate limit. diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs index 566b521dd..6b35810b2 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/IJellyfinDatabaseProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; @@ -50,7 +51,7 @@ public interface IJellyfinDatabaseProvider /// <summary> /// Runs a full Database backup that can later be restored to. /// </summary> - /// <param name="cancellationToken">A cancelation token.</param> + /// <param name="cancellationToken">A cancellation token.</param> /// <returns>A key to identify the backup.</returns> /// <exception cref="NotImplementedException">May throw an NotImplementException if this operation is not supported for this database.</exception> Task<string> MigrationBackupFast(CancellationToken cancellationToken); @@ -59,7 +60,22 @@ public interface IJellyfinDatabaseProvider /// Restores a backup that has been previously created by <see cref="MigrationBackupFast(CancellationToken)"/>. /// </summary> /// <param name="key">The key to the backup from which the current database should be restored from.</param> - /// <param name="cancellationToken">A cancelation token.</param> + /// <param name="cancellationToken">A cancellation token.</param> /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns> Task RestoreBackupFast(string key, CancellationToken cancellationToken); + + /// <summary> + /// Deletes a backup that has been previously created by <see cref="MigrationBackupFast(CancellationToken)"/>. + /// </summary> + /// <param name="key">The key to the backup which should be cleaned up.</param> + /// <returns>A <see cref="Task"/> representing the result of the asynchronous operation.</returns> + Task DeleteBackup(string key); + + /// <summary> + /// Removes all contents from the database. + /// </summary> + /// <param name="dbContext">The Database context.</param> + /// <param name="tableNames">The names of the tables to purge or null for all tables to be purged.</param> + /// <returns>A Task.</returns> + Task PurgeDatabase(JellyfinDbContext dbContext, IEnumerable<string>? tableNames); } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj index 3b619cce6..28c4972d2 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Jellyfin.Database.Implementations.csproj @@ -10,7 +10,21 @@ <Compile Include="..\..\..\SharedVersion.cs" /> </ItemGroup> + <PropertyGroup> + <Authors>Jellyfin Contributors</Authors> + <PackageId>Jellyfin.Database.Implementations</PackageId> + <VersionPrefix>10.11.0</VersionPrefix> + <RepositoryUrl>https://github.com/jellyfin/jellyfin</RepositoryUrl> + <PackageLicenseExpression>GPL-3.0-only</PackageLicenseExpression> + </PropertyGroup> + + <PropertyGroup Condition=" '$(Stability)'=='Unstable'"> + <!-- Include all symbols in the main nupkg until Azure Artifact Feed starts supporting ingesting NuGet symbol packages. --> + <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder> + </PropertyGroup> + <ItemGroup> + <PackageReference Include="Polly" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Relational" /> <PackageReference Include="Microsoft.EntityFrameworkCore.Design"> <PrivateAssets>all</PrivateAssets> diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs index 9db70263d..5163bff8b 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinDbContext.cs @@ -1,9 +1,14 @@ using System; +using System.Data.Common; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Jellyfin.Database.Implementations.Entities; using Jellyfin.Database.Implementations.Entities.Security; using Jellyfin.Database.Implementations.Interfaces; +using Jellyfin.Database.Implementations.Locking; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Logging; namespace Jellyfin.Database.Implementations; @@ -15,7 +20,8 @@ namespace Jellyfin.Database.Implementations; /// <param name="options">The database context options.</param> /// <param name="logger">Logger.</param> /// <param name="jellyfinDatabaseProvider">The provider for the database engine specific operations.</param> -public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILogger<JellyfinDbContext> logger, IJellyfinDatabaseProvider jellyfinDatabaseProvider) : DbContext(options) +/// <param name="entityFrameworkCoreLocking">The locking behavior.</param> +public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILogger<JellyfinDbContext> logger, IJellyfinDatabaseProvider jellyfinDatabaseProvider, IEntityFrameworkCoreLockingBehavior entityFrameworkCoreLocking) : DbContext(options) { /// <summary> /// Gets the <see cref="DbSet{TEntity}"/> containing the access schedules. @@ -157,6 +163,11 @@ public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILog /// </summary> public DbSet<BaseItemTrailerType> BaseItemTrailerTypes => Set<BaseItemTrailerType>(); + /// <summary> + /// Gets the <see cref="DbSet{TEntity}"/>. + /// </summary> + public DbSet<KeyframeData> KeyframeData => Set<KeyframeData>(); + /*public DbSet<Artwork> Artwork => Set<Artwork>(); public DbSet<Book> Books => Set<Book>(); @@ -242,19 +253,41 @@ public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILog public DbSet<TrackMetadata> TrackMetadata => Set<TrackMetadata>();*/ /// <inheritdoc/> - public override int SaveChanges() + public override async Task<int> SaveChangesAsync( + bool acceptAllChangesOnSuccess, + CancellationToken cancellationToken = default) { - foreach (var saveEntity in ChangeTracker.Entries() - .Where(e => e.State == EntityState.Modified) - .Select(entry => entry.Entity) - .OfType<IHasConcurrencyToken>()) + HandleConcurrencyToken(); + + try { - saveEntity.OnSavingChanges(); + var result = -1; + await entityFrameworkCoreLocking.OnSaveChangesAsync(this, async () => + { + result = await base.SaveChangesAsync(acceptAllChangesOnSuccess, cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + return result; + } + catch (Exception e) + { + logger.LogError(e, "Error trying to save changes."); + throw; } + } + + /// <inheritdoc/> + public override int SaveChanges(bool acceptAllChangesOnSuccess) // SaveChanges(bool) is beeing called by SaveChanges() with default to false. + { + HandleConcurrencyToken(); try { - return base.SaveChanges(); + var result = -1; + entityFrameworkCoreLocking.OnSaveChanges(this, () => + { + result = base.SaveChanges(acceptAllChangesOnSuccess); + }); + return result; } catch (Exception e) { @@ -263,6 +296,17 @@ public class JellyfinDbContext(DbContextOptions<JellyfinDbContext> options, ILog } } + private void HandleConcurrencyToken() + { + foreach (var saveEntity in ChangeTracker.Entries() + .Where(e => e.State == EntityState.Modified) + .Select(entry => entry.Entity) + .OfType<IHasConcurrencyToken>()) + { + saveEntity.OnSavingChanges(); + } + } + /// <inheritdoc /> protected override void OnModelCreating(ModelBuilder modelBuilder) { diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinQueryHelperExtensions.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinQueryHelperExtensions.cs new file mode 100644 index 000000000..4d5cfb8c9 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/JellyfinQueryHelperExtensions.cs @@ -0,0 +1,166 @@ +#pragma warning disable RS0030 // Do not use banned APIs + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; +using Jellyfin.Database.Implementations.Entities; +using Microsoft.EntityFrameworkCore; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// Contains a number of query related extensions. +/// </summary> +public static class JellyfinQueryHelperExtensions +{ + private static readonly MethodInfo _containsMethodGenericCache = typeof(Enumerable).GetMethods(BindingFlags.Public | BindingFlags.Static).First(m => m.Name == nameof(Enumerable.Contains) && m.GetParameters().Length == 2); + private static readonly MethodInfo _efParameterInstruction = typeof(EF).GetMethod(nameof(EF.Parameter), BindingFlags.Public | BindingFlags.Static)!; + private static readonly ConcurrentDictionary<Type, MethodInfo> _containsQueryCache = new(); + + /// <summary> + /// Builds an optimised query checking one property against a list of values while maintaining an optimal query. + /// </summary> + /// <typeparam name="TEntity">The entity.</typeparam> + /// <typeparam name="TProperty">The property type to compare.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="oneOf">The list of items to check.</param> + /// <param name="property">Property expression.</param> + /// <returns>A Query.</returns> + public static IQueryable<TEntity> WhereOneOrMany<TEntity, TProperty>(this IQueryable<TEntity> query, IList<TProperty> oneOf, Expression<Func<TEntity, TProperty>> property) + { + return query.Where(OneOrManyExpressionBuilder(oneOf, property)); + } + + /// <summary> + /// Builds a query that checks referenced ItemValues for a cross BaseItem lookup. + /// </summary> + /// <param name="baseQuery">The source query.</param> + /// <param name="context">The database context.</param> + /// <param name="itemValueType">The type of item value to reference.</param> + /// <param name="referenceIds">The list of BaseItem ids to check matches.</param> + /// <param name="invert">If set an exclusion check is performed instead.</param> + /// <returns>A Query.</returns> + public static IQueryable<BaseItemEntity> WhereReferencedItem( + this IQueryable<BaseItemEntity> baseQuery, + JellyfinDbContext context, + ItemValueType itemValueType, + IList<Guid> referenceIds, + bool invert = false) + { + return baseQuery.Where(ReferencedItemFilterExpressionBuilder(context, itemValueType, referenceIds, invert)); + } + + /// <summary> + /// Builds a query expression that checks referenced ItemValues for a cross BaseItem lookup. + /// </summary> + /// <param name="context">The database context.</param> + /// <param name="itemValueType">The type of item value to reference.</param> + /// <param name="referenceIds">The list of BaseItem ids to check matches.</param> + /// <param name="invert">If set an exclusion check is performed instead.</param> + /// <returns>A Query.</returns> + public static Expression<Func<BaseItemEntity, bool>> ReferencedItemFilterExpressionBuilder( + this JellyfinDbContext context, + ItemValueType itemValueType, + IList<Guid> referenceIds, + bool invert = false) + { + // Well genre/artist/album etc items do not actually set the ItemValue of thier specitic types so we cannot match it that way. + /* + "(guid in (select itemid from ItemValues where CleanValue = (select CleanName from TypedBaseItems where guid=@GenreIds and Type=2)))" + */ + + var itemFilter = OneOrManyExpressionBuilder<BaseItemEntity, Guid>(referenceIds, f => f.Id); + + return item => + context.ItemValues + .Join(context.ItemValuesMap, e => e.ItemValueId, e => e.ItemValueId, (item, map) => new { item, map }) + .Any(val => + val.item.Type == itemValueType + && context.BaseItems.Where(itemFilter).Any(e => e.CleanName == val.item.CleanValue) + && val.map.ItemId == item.Id) == EF.Constant(!invert); + } + + /// <summary> + /// Builds an optimised query expression checking one property against a list of values while maintaining an optimal query. + /// </summary> + /// <typeparam name="TEntity">The entity.</typeparam> + /// <typeparam name="TProperty">The property type to compare.</typeparam> + /// <param name="oneOf">The list of items to check.</param> + /// <param name="property">Property expression.</param> + /// <returns>A Query.</returns> + public static Expression<Func<TEntity, bool>> OneOrManyExpressionBuilder<TEntity, TProperty>(this IList<TProperty> oneOf, Expression<Func<TEntity, TProperty>> property) + { + var parameter = Expression.Parameter(typeof(TEntity), "item"); + property = ParameterReplacer.Replace<Func<TEntity, TProperty>, Func<TEntity, TProperty>>(property, property.Parameters[0], parameter); + if (oneOf.Count == 1) + { + var value = oneOf[0]; + if (typeof(TProperty).IsValueType) + { + return Expression.Lambda<Func<TEntity, bool>>(Expression.Equal(property.Body, Expression.Constant(value)), parameter); + } + else + { + return Expression.Lambda<Func<TEntity, bool>>(Expression.ReferenceEqual(property.Body, Expression.Constant(value)), parameter); + } + } + + var containsMethodInfo = _containsQueryCache.GetOrAdd(typeof(TProperty), static (key) => _containsMethodGenericCache.MakeGenericMethod(key)); + + if (oneOf.Count < 4) // arbitrary value choosen. + { + // if we have 3 or fewer values to check against its faster to do a IN(const,const,const) lookup + return Expression.Lambda<Func<TEntity, bool>>(Expression.Call(null, containsMethodInfo, Expression.Constant(oneOf), property.Body), parameter); + } + + return Expression.Lambda<Func<TEntity, bool>>(Expression.Call(null, containsMethodInfo, Expression.Call(null, _efParameterInstruction.MakeGenericMethod(oneOf.GetType()), Expression.Constant(oneOf)), property.Body), parameter); + } + + internal static class ParameterReplacer + { + // Produces an expression identical to 'expression' + // except with 'source' parameter replaced with 'target' expression. + internal static Expression<TOutput> Replace<TInput, TOutput>( + Expression<TInput> expression, + ParameterExpression source, + ParameterExpression target) + { + return new ParameterReplacerVisitor<TOutput>(source, target) + .VisitAndConvert(expression); + } + + private sealed class ParameterReplacerVisitor<TOutput> : ExpressionVisitor + { + private readonly ParameterExpression _source; + private readonly ParameterExpression _target; + + public ParameterReplacerVisitor(ParameterExpression source, ParameterExpression target) + { + _source = source; + _target = target; + } + + internal Expression<TOutput> VisitAndConvert<T>(Expression<T> root) + { + return (Expression<TOutput>)VisitLambda(root); + } + + protected override Expression VisitLambda<T>(Expression<T> node) + { + // Leave all parameters alone except the one we want to replace. + var parameters = node.Parameters.Select(p => p == _source ? _target : p); + + return Expression.Lambda<TOutput>(Visit(node.Body), parameters); + } + + protected override Expression VisitParameter(ParameterExpression node) + { + // Replace the source with the target, visit other params as usual. + return node == _source ? _target : base.VisitParameter(node); + } + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs new file mode 100644 index 000000000..465c31212 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/IEntityFrameworkCoreLockingBehavior.cs @@ -0,0 +1,32 @@ +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Defines a jellyfin locking behavior that can be configured. +/// </summary> +public interface IEntityFrameworkCoreLockingBehavior +{ + /// <summary> + /// Provides access to the builder to setup any connection related locking behavior. + /// </summary> + /// <param name="optionsBuilder">The options builder.</param> + void Initialise(DbContextOptionsBuilder optionsBuilder); + + /// <summary> + /// Will be invoked when changes should be saved in the current locking behavior. + /// </summary> + /// <param name="context">The database context invoking the action.</param> + /// <param name="saveChanges">Callback for performing the actual save changes.</param> + void OnSaveChanges(JellyfinDbContext context, Action saveChanges); + + /// <summary> + /// Will be invoked when changes should be saved in the current locking behavior. + /// </summary> + /// <param name="context">The database context invoking the action.</param> + /// <param name="saveChanges">Callback for performing the actual save changes.</param> + /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns> + Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges); +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs new file mode 100644 index 000000000..3b654f4c4 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/NoLockBehavior.cs @@ -0,0 +1,41 @@ +using System; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Default lock behavior. Defines no explicit application locking behavior. +/// </summary> +public class NoLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly ILogger<NoLockBehavior> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="NoLockBehavior"/> class. + /// </summary> + /// <param name="logger">The Application logger.</param> + public NoLockBehavior(ILogger<NoLockBehavior> logger) + { + _logger = logger; + } + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + saveChanges(); + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: NoLock."); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + await saveChanges().ConfigureAwait(false); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs new file mode 100644 index 000000000..9395b2e2d --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/OptimisticLockBehavior.cs @@ -0,0 +1,137 @@ +using System; +using System.Data.Common; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; +using Polly; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// Defines a locking mechanism that will retry any write operation for a few times. +/// </summary> +public class OptimisticLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly Policy _writePolicy; + private readonly AsyncPolicy _writeAsyncPolicy; + private readonly ILogger<OptimisticLockBehavior> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="OptimisticLockBehavior"/> class. + /// </summary> + /// <param name="logger">The application logger.</param> + public OptimisticLockBehavior(ILogger<OptimisticLockBehavior> logger) + { + TimeSpan[] sleepDurations = [ + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(50), + TimeSpan.FromMilliseconds(250), + TimeSpan.FromMilliseconds(150), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(3) + ]; + _logger = logger; + _writePolicy = Policy.HandleInner<Exception>(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetry(sleepDurations, RetryHandle); + _writeAsyncPolicy = Policy.HandleInner<Exception>(e => e.Message.Contains("database is locked", StringComparison.InvariantCultureIgnoreCase)).WaitAndRetryAsync(sleepDurations, RetryHandle); + + void RetryHandle(Exception exception, TimeSpan timespan, int retryNo, Context context) + { + if (retryNo < sleepDurations.Length) + { + _logger.LogWarning("Operation failed retry {RetryNo}", retryNo); + } + else + { + _logger.LogError(exception, "Operation failed retry {RetryNo}", retryNo); + } + } + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: Optimistic."); + optionsBuilder.AddInterceptors(new RetryInterceptor(_writeAsyncPolicy, _writePolicy)); + optionsBuilder.AddInterceptors(new TransactionLockingInterceptor(_writeAsyncPolicy, _writePolicy)); + } + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + _writePolicy.ExecuteAndCapture(saveChanges); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + await _writeAsyncPolicy.ExecuteAndCaptureAsync(saveChanges).ConfigureAwait(false); + } + + private sealed class TransactionLockingInterceptor : DbTransactionInterceptor + { + private readonly AsyncPolicy _asyncRetryPolicy; + private readonly Policy _retryPolicy; + + public TransactionLockingInterceptor(AsyncPolicy asyncRetryPolicy, Policy retryPolicy) + { + _asyncRetryPolicy = asyncRetryPolicy; + _retryPolicy = retryPolicy; + } + + public override InterceptionResult<DbTransaction> TransactionStarting(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result) + { + return InterceptionResult<DbTransaction>.SuppressWithResult(_retryPolicy.Execute(() => connection.BeginTransaction(eventData.IsolationLevel))); + } + + public override async ValueTask<InterceptionResult<DbTransaction>> TransactionStartingAsync(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<DbTransaction>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await connection.BeginTransactionAsync(eventData.IsolationLevel, cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + } + + private sealed class RetryInterceptor : DbCommandInterceptor + { + private readonly AsyncPolicy _asyncRetryPolicy; + private readonly Policy _retryPolicy; + + public RetryInterceptor(AsyncPolicy asyncRetryPolicy, Policy retryPolicy) + { + _asyncRetryPolicy = asyncRetryPolicy; + _retryPolicy = retryPolicy; + } + + public override InterceptionResult<int> NonQueryExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<int> result) + { + return InterceptionResult<int>.SuppressWithResult(_retryPolicy.Execute(command.ExecuteNonQuery)); + } + + public override async ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<int>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + + public override InterceptionResult<object> ScalarExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<object> result) + { + return InterceptionResult<object>.SuppressWithResult(_retryPolicy.Execute(() => command.ExecuteScalar()!)); + } + + public override async ValueTask<InterceptionResult<object>> ScalarExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<object> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<object>.SuppressWithResult((await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false)!).ConfigureAwait(false))!); + } + + public override InterceptionResult<DbDataReader> ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(_retryPolicy.Execute(command.ExecuteReader)); + } + + public override async ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result, CancellationToken cancellationToken = default) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(await _asyncRetryPolicy.ExecuteAsync(async () => await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false)); + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs new file mode 100644 index 000000000..2d6bc6902 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/Locking/PessimisticLockBehavior.cs @@ -0,0 +1,296 @@ +#pragma warning disable MT1013 // Releasing lock without guarantee of execution +#pragma warning disable MT1012 // Acquiring lock without guarantee of releasing + +using System; +using System.Data; +using System.Data.Common; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Database.Implementations.Locking; + +/// <summary> +/// A locking behavior that will always block any operation while a write is requested. Mimicks the old SqliteRepository behavior. +/// </summary> +public class PessimisticLockBehavior : IEntityFrameworkCoreLockingBehavior +{ + private readonly ILogger<PessimisticLockBehavior> _logger; + private readonly ILoggerFactory _loggerFactory; + + /// <summary> + /// Initializes a new instance of the <see cref="PessimisticLockBehavior"/> class. + /// </summary> + /// <param name="logger">The application logger.</param> + /// <param name="loggerFactory">The logger factory.</param> + public PessimisticLockBehavior(ILogger<PessimisticLockBehavior> logger, ILoggerFactory loggerFactory) + { + _logger = logger; + _loggerFactory = loggerFactory; + } + + private static ReaderWriterLockSlim DatabaseLock { get; } = new(LockRecursionPolicy.SupportsRecursion); + + /// <inheritdoc/> + public void OnSaveChanges(JellyfinDbContext context, Action saveChanges) + { + using (DbLock.EnterWrite(_logger)) + { + saveChanges(); + } + } + + /// <inheritdoc/> + public void Initialise(DbContextOptionsBuilder optionsBuilder) + { + _logger.LogInformation("The database locking mode has been set to: Pessimistic."); + optionsBuilder.AddInterceptors(new CommandLockingInterceptor(_loggerFactory.CreateLogger<CommandLockingInterceptor>())); + optionsBuilder.AddInterceptors(new TransactionLockingInterceptor(_loggerFactory.CreateLogger<TransactionLockingInterceptor>())); + } + + /// <inheritdoc/> + public async Task OnSaveChangesAsync(JellyfinDbContext context, Func<Task> saveChanges) + { + using (DbLock.EnterWrite(_logger)) + { + await saveChanges().ConfigureAwait(false); + } + } + + private sealed class TransactionLockingInterceptor : DbTransactionInterceptor + { + private readonly ILogger _logger; + + public TransactionLockingInterceptor(ILogger logger) + { + _logger = logger; + } + + public override InterceptionResult<DbTransaction> TransactionStarting(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result) + { + DbLock.BeginWriteLock(_logger); + + return base.TransactionStarting(connection, eventData, result); + } + + public override ValueTask<InterceptionResult<DbTransaction>> TransactionStartingAsync(DbConnection connection, TransactionStartingEventData eventData, InterceptionResult<DbTransaction> result, CancellationToken cancellationToken = default) + { + DbLock.BeginWriteLock(_logger); + + return base.TransactionStartingAsync(connection, eventData, result, cancellationToken); + } + + public override void TransactionCommitted(DbTransaction transaction, TransactionEndEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionCommitted(transaction, eventData); + } + + public override Task TransactionCommittedAsync(DbTransaction transaction, TransactionEndEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionCommittedAsync(transaction, eventData, cancellationToken); + } + + public override void TransactionFailed(DbTransaction transaction, TransactionErrorEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionFailed(transaction, eventData); + } + + public override Task TransactionFailedAsync(DbTransaction transaction, TransactionErrorEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionFailedAsync(transaction, eventData, cancellationToken); + } + + public override void TransactionRolledBack(DbTransaction transaction, TransactionEndEventData eventData) + { + DbLock.EndWriteLock(_logger); + + base.TransactionRolledBack(transaction, eventData); + } + + public override Task TransactionRolledBackAsync(DbTransaction transaction, TransactionEndEventData eventData, CancellationToken cancellationToken = default) + { + DbLock.EndWriteLock(_logger); + + return base.TransactionRolledBackAsync(transaction, eventData, cancellationToken); + } + } + + /// <summary> + /// Adds strict read/write locking. + /// </summary> + private sealed class CommandLockingInterceptor : DbCommandInterceptor + { + private readonly ILogger _logger; + + public CommandLockingInterceptor(ILogger logger) + { + _logger = logger; + } + + public override InterceptionResult<int> NonQueryExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<int> result) + { + using (DbLock.EnterWrite(_logger, command)) + { + return InterceptionResult<int>.SuppressWithResult(command.ExecuteNonQuery()); + } + } + + public override async ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<int> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterWrite(_logger, command)) + { + return InterceptionResult<int>.SuppressWithResult(await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false)); + } + } + + public override InterceptionResult<object> ScalarExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<object> result) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<object>.SuppressWithResult(command.ExecuteScalar()!); + } + } + + public override async ValueTask<InterceptionResult<object>> ScalarExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<object> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<object>.SuppressWithResult((await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!); + } + } + + public override InterceptionResult<DbDataReader> ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(command.ExecuteReader()!); + } + } + + public override async ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result, CancellationToken cancellationToken = default) + { + using (DbLock.EnterRead(_logger)) + { + return InterceptionResult<DbDataReader>.SuppressWithResult(await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)); + } + } + } + + private sealed class DbLock : IDisposable + { + private readonly Action? _action; + private bool _disposed; + + private static readonly IDisposable _noLock = new DbLock(null) { _disposed = true }; + private static (string Command, Guid Id, DateTimeOffset QueryDate, bool Printed) _blockQuery; + + public DbLock(Action? action = null) + { + _action = action; + } + +#pragma warning disable IDISP015 // Member should not return created and cached instance + public static IDisposable EnterWrite(ILogger logger, IDbCommand? command = null, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) +#pragma warning restore IDISP015 // Member should not return created and cached instance + { + logger.LogTrace("Enter Write for {Caller}:{Line}", callerMemberName, callerNo); + if (DatabaseLock.IsWriteLockHeld) + { + logger.LogTrace("Write Held {Caller}:{Line}", callerMemberName, callerNo); + return _noLock; + } + + BeginWriteLock(logger, command, callerMemberName, callerNo); + return new DbLock(() => + { + EndWriteLock(logger, callerMemberName, callerNo); + }); + } + +#pragma warning disable IDISP015 // Member should not return created and cached instance + public static IDisposable EnterRead(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) +#pragma warning restore IDISP015 // Member should not return created and cached instance + { + logger.LogTrace("Enter Read {Caller}:{Line}", callerMemberName, callerNo); + if (DatabaseLock.IsWriteLockHeld) + { + logger.LogTrace("Write Held {Caller}:{Line}", callerMemberName, callerNo); + return _noLock; + } + + BeginReadLock(logger, callerMemberName, callerNo); + return new DbLock(() => + { + ExitReadLock(logger, callerMemberName, callerNo); + }); + } + + public static void BeginWriteLock(ILogger logger, IDbCommand? command = null, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Aquire Write {Caller}:{Line}", callerMemberName, callerNo); + if (!DatabaseLock.TryEnterWriteLock(TimeSpan.FromMilliseconds(1000))) + { + var blockingQuery = _blockQuery; + if (!blockingQuery.Printed) + { + _blockQuery = (blockingQuery.Command, blockingQuery.Id, blockingQuery.QueryDate, true); + logger.LogInformation("QueryLock: {Id} --- {Query}", blockingQuery.Id, blockingQuery.Command); + } + + logger.LogInformation("Query congestion detected: '{Id}' since '{Date}'", blockingQuery.Id, blockingQuery.QueryDate); + + DatabaseLock.EnterWriteLock(); + + logger.LogInformation("Query congestion cleared: '{Id}' for '{Date}'", blockingQuery.Id, DateTimeOffset.Now - blockingQuery.QueryDate); + } + + _blockQuery = (command?.CommandText ?? "Transaction", Guid.NewGuid(), DateTimeOffset.Now, false); + + logger.LogTrace("Write Aquired {Caller}:{Line}", callerMemberName, callerNo); + } + + public static void BeginReadLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Aquire Write {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.EnterReadLock(); + logger.LogTrace("Read Aquired {Caller}:{Line}", callerMemberName, callerNo); + } + + public static void EndWriteLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Release Write {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.ExitWriteLock(); + } + + public static void ExitReadLock(ILogger logger, [CallerMemberName] string? callerMemberName = null, [CallerLineNumber] int? callerNo = null) + { + logger.LogTrace("Release Read {Caller}:{Line}", callerMemberName, callerNo); + DatabaseLock.ExitReadLock(); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (_action is not null) + { + _action(); + } + } + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/AncestorIdConfiguration.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/AncestorIdConfiguration.cs index 1cb4a1eb1..67269153d 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/AncestorIdConfiguration.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/AncestorIdConfiguration.cs @@ -14,7 +14,7 @@ public class AncestorIdConfiguration : IEntityTypeConfiguration<AncestorId> { builder.HasKey(e => new { e.ItemId, e.ParentItemId }); builder.HasIndex(e => e.ParentItemId); - builder.HasOne(e => e.ParentItem).WithMany(e => e.ParentAncestors).HasForeignKey(f => f.ParentItemId); - builder.HasOne(e => e.Item).WithMany(e => e.Children).HasForeignKey(f => f.ItemId); + builder.HasOne(e => e.ParentItem).WithMany(e => e.Children).HasForeignKey(f => f.ParentItemId); + builder.HasOne(e => e.Item).WithMany(e => e.Parents).HasForeignKey(f => f.ItemId); } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/BaseItemConfiguration.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/BaseItemConfiguration.cs index 37816faec..4a76113bf 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/BaseItemConfiguration.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/BaseItemConfiguration.cs @@ -24,7 +24,7 @@ public class BaseItemConfiguration : IEntityTypeConfiguration<BaseItemEntity> builder.HasMany(e => e.MediaStreams); builder.HasMany(e => e.Chapters); builder.HasMany(e => e.Provider); - builder.HasMany(e => e.ParentAncestors); + builder.HasMany(e => e.Parents); builder.HasMany(e => e.Children); builder.HasMany(e => e.LockedFields); builder.HasMany(e => e.TrailerTypes); diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/ItemValuesConfiguration.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/ItemValuesConfiguration.cs index c8e003eaa..97ebc2e01 100644 --- a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/ItemValuesConfiguration.cs +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/ItemValuesConfiguration.cs @@ -13,6 +13,7 @@ public class ItemValuesConfiguration : IEntityTypeConfiguration<ItemValue> public void Configure(EntityTypeBuilder<ItemValue> builder) { builder.HasKey(e => e.ItemValueId); - builder.HasIndex(e => new { e.Type, e.CleanValue }).IsUnique(); + builder.HasIndex(e => new { e.Type, e.CleanValue }); + builder.HasIndex(e => new { e.Type, e.Value }).IsUnique(); } } diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/KeyframeDataConfiguration.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/KeyframeDataConfiguration.cs new file mode 100644 index 000000000..3f5d458ca --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ModelConfiguration/KeyframeDataConfiguration.cs @@ -0,0 +1,18 @@ +using Jellyfin.Database.Implementations.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Jellyfin.Database.Implementations.ModelConfiguration; + +/// <summary> +/// KeyframeData Configuration. +/// </summary> +public class KeyframeDataConfiguration : IEntityTypeConfiguration<KeyframeData> +{ + /// <inheritdoc/> + public void Configure(EntityTypeBuilder<KeyframeData> builder) + { + builder.HasKey(e => e.ItemId); + builder.HasOne(e => e.Item).WithMany().HasForeignKey(e => e.ItemId); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs new file mode 100644 index 000000000..7654dd3c5 --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/ProgressablePartitionReporting.cs @@ -0,0 +1,55 @@ +using System; +using System.Diagnostics; +using System.Linq; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// Wrapper for progress reporting on Partition helpers. +/// </summary> +/// <typeparam name="TEntity">The entity to load.</typeparam> +public class ProgressablePartitionReporting<TEntity> +{ + private readonly IOrderedQueryable<TEntity> _source; + + private readonly Stopwatch _partitionTime = new(); + + private readonly Stopwatch _itemTime = new(); + + internal ProgressablePartitionReporting(IOrderedQueryable<TEntity> source) + { + _source = source; + } + + internal Action<TEntity, int, int>? OnBeginItem { get; set; } + + internal Action<int>? OnBeginPartition { get; set; } + + internal Action<TEntity, int, int, TimeSpan>? OnEndItem { get; set; } + + internal Action<int, TimeSpan>? OnEndPartition { get; set; } + + internal IOrderedQueryable<TEntity> Source => _source; + + internal void BeginItem(TEntity entity, int iteration, int itemIndex) + { + _itemTime.Restart(); + OnBeginItem?.Invoke(entity, iteration, itemIndex); + } + + internal void BeginPartition(int iteration) + { + _partitionTime.Restart(); + OnBeginPartition?.Invoke(iteration); + } + + internal void EndItem(TEntity entity, int iteration, int itemIndex) + { + OnEndItem?.Invoke(entity, iteration, itemIndex, _itemTime.Elapsed); + } + + internal void EndPartition(int iteration) + { + OnEndPartition?.Invoke(iteration, _partitionTime.Elapsed); + } +} diff --git a/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs b/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs new file mode 100644 index 000000000..bb66bddca --- /dev/null +++ b/src/Jellyfin.Database/Jellyfin.Database.Implementations/QueryPartitionHelpers.cs @@ -0,0 +1,215 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace Jellyfin.Database.Implementations; + +/// <summary> +/// Contains helpers to partition EFCore queries. +/// </summary> +public static class QueryPartitionHelpers +{ + /// <summary> + /// Adds a callback to any directly following calls of Partition for every partition thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="beginPartition">The callback invoked for partition before enumerating items.</param> + /// <param name="endPartition">The callback invoked for partition after enumerating items.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithPartitionProgress<TEntity>(this IOrderedQueryable<TEntity> query, Action<int>? beginPartition = null, Action<int, TimeSpan>? endPartition = null) + { + var progressable = new ProgressablePartitionReporting<TEntity>(query); + progressable.OnBeginPartition = beginPartition; + progressable.OnEndPartition = endPartition; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every item thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="beginItem">The callback invoked for each item before processing.</param> + /// <param name="endItem">The callback invoked for each item after processing.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithItemProgress<TEntity>(this IOrderedQueryable<TEntity> query, Action<TEntity, int, int>? beginItem = null, Action<TEntity, int, int, TimeSpan>? endItem = null) + { + var progressable = new ProgressablePartitionReporting<TEntity>(query); + progressable.OnBeginItem = beginItem; + progressable.OnEndItem = endItem; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every partition thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="progressable">The source query.</param> + /// <param name="beginPartition">The callback invoked for partition before enumerating items.</param> + /// <param name="endPartition">The callback invoked for partition after enumerating items.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithPartitionProgress<TEntity>(this ProgressablePartitionReporting<TEntity> progressable, Action<int>? beginPartition = null, Action<int, TimeSpan>? endPartition = null) + { + progressable.OnBeginPartition = beginPartition; + progressable.OnEndPartition = endPartition; + return progressable; + } + + /// <summary> + /// Adds a callback to any directly following calls of Partition for every item thats been invoked. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="progressable">The source query.</param> + /// <param name="beginItem">The callback invoked for each item before processing.</param> + /// <param name="endItem">The callback invoked for each item after processing.</param> + /// <returns>A queryable that can be used to partition.</returns> + public static ProgressablePartitionReporting<TEntity> WithItemProgress<TEntity>(this ProgressablePartitionReporting<TEntity> progressable, Action<TEntity, int, int>? beginItem = null, Action<TEntity, int, int, TimeSpan>? endItem = null) + { + progressable.OnBeginItem = beginItem; + progressable.OnEndItem = endItem; + return progressable; + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions in a lazy manner reading each item from the database as its requested. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="partitionInfo">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionAsync<TEntity>(this ProgressablePartitionReporting<TEntity> partitionInfo, int partitionSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var item in partitionInfo.Source.PartitionAsync(partitionSize, partitionInfo, cancellationToken).ConfigureAwait(false)) + { + yield return item; + } + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions directly into memory. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="partitionInfo">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionEagerAsync<TEntity>(this ProgressablePartitionReporting<TEntity> partitionInfo, int partitionSize, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var item in partitionInfo.Source.PartitionEagerAsync(partitionSize, partitionInfo, cancellationToken).ConfigureAwait(false)) + { + yield return item; + } + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions in a lazy manner reading each item from the database as its requested. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="progressablePartition">Reporting helper.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionAsync<TEntity>( + this IOrderedQueryable<TEntity> query, + int partitionSize, + ProgressablePartitionReporting<TEntity>? progressablePartition = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var iterator = 0; + int itemCounter; + do + { + progressablePartition?.BeginPartition(iterator); + itemCounter = 0; + await foreach (var item in query + .Skip(partitionSize * iterator) + .Take(partitionSize) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + progressablePartition?.BeginItem(item, iterator, itemCounter); + yield return item; + progressablePartition?.EndItem(item, iterator, itemCounter); + itemCounter++; + } + + progressablePartition?.EndPartition(iterator); + iterator++; + } while (itemCounter == partitionSize && !cancellationToken.IsCancellationRequested); + } + + /// <summary> + /// Enumerates the source query by loading the entities in partitions directly into memory. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <param name="partitionSize">The number of elements to load per partition.</param> + /// <param name="progressablePartition">Reporting helper.</param> + /// <param name="cancellationToken">The cancelation token.</param> + /// <returns>A enumerable representing the whole of the query.</returns> + public static async IAsyncEnumerable<TEntity> PartitionEagerAsync<TEntity>( + this IOrderedQueryable<TEntity> query, + int partitionSize, + ProgressablePartitionReporting<TEntity>? progressablePartition = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var iterator = 0; + int itemCounter; + var items = ArrayPool<TEntity>.Shared.Rent(partitionSize); + try + { + do + { + progressablePartition?.BeginPartition(iterator); + itemCounter = 0; + await foreach (var item in query + .Skip(partitionSize * iterator) + .Take(partitionSize) + .AsAsyncEnumerable() + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + items[itemCounter++] = item; + } + + for (int i = 0; i < itemCounter; i++) + { + progressablePartition?.BeginItem(items[i], iterator, itemCounter); + yield return items[i]; + progressablePartition?.EndItem(items[i], iterator, itemCounter); + } + + progressablePartition?.EndPartition(iterator); + iterator++; + } while (itemCounter == partitionSize && !cancellationToken.IsCancellationRequested); + } + finally + { + ArrayPool<TEntity>.Shared.Return(items); + } + } + + /// <summary> + /// Adds an Index to the enumeration of the async enumerable. + /// </summary> + /// <typeparam name="TEntity">The entity to load.</typeparam> + /// <param name="query">The source query.</param> + /// <returns>The source list with an index added.</returns> + public static async IAsyncEnumerable<(TEntity Item, int Index)> WithIndex<TEntity>(this IAsyncEnumerable<TEntity> query) + { + var index = 0; + await foreach (var item in query.ConfigureAwait(false)) + { + yield return (item, index++); + } + } +} |