diff options
26 files changed, 387 insertions, 135 deletions
diff --git a/Directory.Packages.props b/Directory.Packages.props index 8d3dd064b..755d1427f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,7 @@ <PackageVersion Include="prometheus-net" Version="8.2.1" /> <PackageVersion Include="Serilog.AspNetCore" Version="8.0.1" /> <PackageVersion Include="Serilog.Enrichers.Thread" Version="3.1.0" /> - <PackageVersion Include="Serilog.Settings.Configuration" Version="8.0.0" /> + <PackageVersion Include="Serilog.Settings.Configuration" Version="8.0.1" /> <PackageVersion Include="Serilog.Sinks.Async" Version="1.5.0" /> <PackageVersion Include="Serilog.Sinks.Console" Version="5.0.1" /> <PackageVersion Include="Serilog.Sinks.File" Version="5.0.0" /> diff --git a/Emby.Server.Implementations/Data/BaseSqliteRepository.cs b/Emby.Server.Implementations/Data/BaseSqliteRepository.cs index ee1598e7f..5291999dc 100644 --- a/Emby.Server.Implementations/Data/BaseSqliteRepository.cs +++ b/Emby.Server.Implementations/Data/BaseSqliteRepository.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Threading; using Jellyfin.Extensions; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; @@ -13,6 +14,8 @@ namespace Emby.Server.Implementations.Data public abstract class BaseSqliteRepository : IDisposable { private bool _disposed = false; + private SemaphoreSlim _writeLock = new SemaphoreSlim(1, 1); + private SqliteConnection _writeConnection; /// <summary> /// Initializes a new instance of the <see cref="BaseSqliteRepository"/> class. @@ -87,9 +90,55 @@ namespace Emby.Server.Implementations.Data } } - protected SqliteConnection GetConnection() + protected ManagedConnection GetConnection(bool readOnly = false) { - var connection = new SqliteConnection($"Filename={DbFilePath}"); + if (!readOnly) + { + _writeLock.Wait(); + if (_writeConnection is not null) + { + return new ManagedConnection(_writeConnection, _writeLock); + } + + var writeConnection = new SqliteConnection($"Filename={DbFilePath};Pooling=False"); + writeConnection.Open(); + + if (CacheSize.HasValue) + { + writeConnection.Execute("PRAGMA cache_size=" + CacheSize.Value); + } + + if (!string.IsNullOrWhiteSpace(LockingMode)) + { + writeConnection.Execute("PRAGMA locking_mode=" + LockingMode); + } + + if (!string.IsNullOrWhiteSpace(JournalMode)) + { + writeConnection.Execute("PRAGMA journal_mode=" + JournalMode); + } + + if (JournalSizeLimit.HasValue) + { + writeConnection.Execute("PRAGMA journal_size_limit=" + JournalSizeLimit.Value); + } + + if (Synchronous.HasValue) + { + writeConnection.Execute("PRAGMA synchronous=" + (int)Synchronous.Value); + } + + if (PageSize.HasValue) + { + writeConnection.Execute("PRAGMA page_size=" + PageSize.Value); + } + + writeConnection.Execute("PRAGMA temp_store=" + (int)TempStore); + + return new ManagedConnection(_writeConnection = writeConnection, _writeLock); + } + + var connection = new SqliteConnection($"Filename={DbFilePath};Mode=ReadOnly"); connection.Open(); if (CacheSize.HasValue) @@ -124,17 +173,17 @@ namespace Emby.Server.Implementations.Data connection.Execute("PRAGMA temp_store=" + (int)TempStore); - return connection; + return new ManagedConnection(connection, null); } - public SqliteCommand PrepareStatement(SqliteConnection connection, string sql) + public SqliteCommand PrepareStatement(ManagedConnection connection, string sql) { var command = connection.CreateCommand(); command.CommandText = sql; return command; } - protected bool TableExists(SqliteConnection connection, string name) + protected bool TableExists(ManagedConnection connection, string name) { using var statement = PrepareStatement(connection, "select DISTINCT tbl_name from sqlite_master"); foreach (var row in statement.ExecuteQuery()) @@ -148,7 +197,7 @@ namespace Emby.Server.Implementations.Data return false; } - protected List<string> GetColumnNames(SqliteConnection connection, string table) + protected List<string> GetColumnNames(ManagedConnection connection, string table) { var columnNames = new List<string>(); @@ -163,7 +212,7 @@ namespace Emby.Server.Implementations.Data return columnNames; } - protected void AddColumn(SqliteConnection connection, string table, string columnName, string type, List<string> existingColumnNames) + protected void AddColumn(ManagedConnection connection, string table, string columnName, string type, List<string> existingColumnNames) { if (existingColumnNames.Contains(columnName, StringComparison.OrdinalIgnoreCase)) { @@ -196,6 +245,24 @@ namespace Emby.Server.Implementations.Data return; } + if (dispose) + { + _writeLock.Wait(); + try + { + _writeConnection.Dispose(); + } + finally + { + _writeLock.Release(); + } + + _writeLock.Dispose(); + } + + _writeConnection = null; + _writeLock = null; + _disposed = true; } } diff --git a/Emby.Server.Implementations/Data/ManagedConnection.cs b/Emby.Server.Implementations/Data/ManagedConnection.cs new file mode 100644 index 000000000..860950b30 --- /dev/null +++ b/Emby.Server.Implementations/Data/ManagedConnection.cs @@ -0,0 +1,62 @@ +#pragma warning disable CS1591 + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Data.Sqlite; + +namespace Emby.Server.Implementations.Data; + +public sealed class ManagedConnection : IDisposable +{ + private readonly SemaphoreSlim? _writeLock; + + private SqliteConnection _db; + + private bool _disposed = false; + + public ManagedConnection(SqliteConnection db, SemaphoreSlim? writeLock) + { + _db = db; + _writeLock = writeLock; + } + + public SqliteTransaction BeginTransaction() + => _db.BeginTransaction(); + + public SqliteCommand CreateCommand() + => _db.CreateCommand(); + + public void Execute(string commandText) + => _db.Execute(commandText); + + public SqliteCommand PrepareStatement(string sql) + => _db.PrepareStatement(sql); + + public IEnumerable<SqliteDataReader> Query(string commandText) + => _db.Query(commandText); + + public void Dispose() + { + if (_disposed) + { + return; + } + + if (_writeLock is null) + { + // Read connections are managed with an internal pool + _db.Dispose(); + } + else + { + // Write lock is managed by BaseSqliteRepository + // Don't dispose here + _writeLock.Release(); + } + + _db = null!; + + _disposed = true; + } +} diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index a4c6b2d31..81ee55d26 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -600,7 +600,7 @@ namespace Emby.Server.Implementations.Data transaction.Commit(); } - private void SaveItemsInTransaction(SqliteConnection db, IEnumerable<(BaseItem Item, List<Guid> AncestorIds, BaseItem TopParent, string UserDataKey, List<string> InheritedTags)> tuples) + private void SaveItemsInTransaction(ManagedConnection db, IEnumerable<(BaseItem Item, List<Guid> AncestorIds, BaseItem TopParent, string UserDataKey, List<string> InheritedTags)> tuples) { using (var saveItemStatement = PrepareStatement(db, SaveItemCommandText)) using (var deleteAncestorsStatement = PrepareStatement(db, "delete from AncestorIds where ItemId=@ItemId")) @@ -1260,7 +1260,7 @@ namespace Emby.Server.Implementations.Data CheckDisposed(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, _retrieveItemColumnsSelectQuery)) { statement.TryBind("@guid", id); @@ -1886,7 +1886,7 @@ namespace Emby.Server.Implementations.Data CheckDisposed(); var chapters = new List<ChapterInfo>(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, "select StartPositionTicks,Name,ImagePath,ImageDateModified from " + ChaptersTableName + " where ItemId = @ItemId order by ChapterIndex asc")) { statement.TryBind("@ItemId", item.Id); @@ -1905,7 +1905,7 @@ namespace Emby.Server.Implementations.Data { CheckDisposed(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, "select StartPositionTicks,Name,ImagePath,ImageDateModified from " + ChaptersTableName + " where ItemId = @ItemId and ChapterIndex=@ChapterIndex")) { statement.TryBind("@ItemId", item.Id); @@ -1979,7 +1979,7 @@ namespace Emby.Server.Implementations.Data transaction.Commit(); } - private void InsertChapters(Guid idBlob, IReadOnlyList<ChapterInfo> chapters, SqliteConnection db) + private void InsertChapters(Guid idBlob, IReadOnlyList<ChapterInfo> chapters, ManagedConnection db) { var startIndex = 0; var limit = 100; @@ -2468,7 +2468,7 @@ namespace Emby.Server.Implementations.Data var commandText = commandTextBuilder.ToString(); using (new QueryTimeLogger(Logger, commandText)) - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText)) { if (EnableJoinUserData(query)) @@ -2536,7 +2536,7 @@ namespace Emby.Server.Implementations.Data var commandText = commandTextBuilder.ToString(); var items = new List<BaseItem>(); using (new QueryTimeLogger(Logger, commandText)) - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText)) { if (EnableJoinUserData(query)) @@ -2744,7 +2744,7 @@ namespace Emby.Server.Implementations.Data var list = new List<BaseItem>(); var result = new QueryResult<BaseItem>(); - using var connection = GetConnection(); + using var connection = GetConnection(true); using var transaction = connection.BeginTransaction(); if (!isReturningZeroItems) { @@ -2926,7 +2926,7 @@ namespace Emby.Server.Implementations.Data var commandText = commandTextBuilder.ToString(); var list = new List<Guid>(); using (new QueryTimeLogger(Logger, commandText)) - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText)) { if (EnableJoinUserData(query)) @@ -4475,7 +4475,7 @@ where AncestorIdText not null and ItemValues.Value not null and ItemValues.Type transaction.Commit(); } - private void ExecuteWithSingleParam(SqliteConnection db, string query, Guid value) + private void ExecuteWithSingleParam(ManagedConnection db, string query, Guid value) { using (var statement = PrepareStatement(db, query)) { @@ -4508,7 +4508,7 @@ where AncestorIdText not null and ItemValues.Value not null and ItemValues.Type } var list = new List<string>(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText.ToString())) { // Run this again to bind the params @@ -4546,7 +4546,7 @@ where AncestorIdText not null and ItemValues.Value not null and ItemValues.Type } var list = new List<PersonInfo>(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText.ToString())) { // Run this again to bind the params @@ -4631,7 +4631,7 @@ AND Type = @InternalPersonType)"); return whereClauses; } - private void UpdateAncestors(Guid itemId, List<Guid> ancestorIds, SqliteConnection db, SqliteCommand deleteAncestorsStatement) + private void UpdateAncestors(Guid itemId, List<Guid> ancestorIds, ManagedConnection db, SqliteCommand deleteAncestorsStatement) { if (itemId.IsEmpty()) { @@ -4786,7 +4786,7 @@ AND Type = @InternalPersonType)"); var list = new List<string>(); using (new QueryTimeLogger(Logger, commandText)) - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, commandText)) { foreach (var row in statement.ExecuteQuery()) @@ -4986,8 +4986,8 @@ AND Type = @InternalPersonType)"); var list = new List<(BaseItem, ItemCounts)>(); var result = new QueryResult<(BaseItem, ItemCounts)>(); using (new QueryTimeLogger(Logger, commandText)) - using (var connection = GetConnection()) - using (var transaction = connection.BeginTransaction(deferred: true)) + using (var connection = GetConnection(true)) + using (var transaction = connection.BeginTransaction()) { if (!isReturningZeroItems) { @@ -5147,7 +5147,7 @@ AND Type = @InternalPersonType)"); return list; } - private void UpdateItemValues(Guid itemId, List<(int MagicNumber, string Value)> values, SqliteConnection db) + private void UpdateItemValues(Guid itemId, List<(int MagicNumber, string Value)> values, ManagedConnection db) { if (itemId.IsEmpty()) { @@ -5166,7 +5166,7 @@ AND Type = @InternalPersonType)"); InsertItemValues(itemId, values, db); } - private void InsertItemValues(Guid id, List<(int MagicNumber, string Value)> values, SqliteConnection db) + private void InsertItemValues(Guid id, List<(int MagicNumber, string Value)> values, ManagedConnection db) { const int Limit = 100; var startIndex = 0; @@ -5238,7 +5238,7 @@ AND Type = @InternalPersonType)"); transaction.Commit(); } - private void InsertPeople(Guid id, List<PersonInfo> people, SqliteConnection db) + private void InsertPeople(Guid id, List<PersonInfo> people, ManagedConnection db) { const int Limit = 100; var startIndex = 0; @@ -5334,7 +5334,7 @@ AND Type = @InternalPersonType)"); cmdText += " order by StreamIndex ASC"; - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) { var list = new List<MediaStream>(); @@ -5387,7 +5387,7 @@ AND Type = @InternalPersonType)"); transaction.Commit(); } - private void InsertMediaStreams(Guid id, IReadOnlyList<MediaStream> streams, SqliteConnection db) + private void InsertMediaStreams(Guid id, IReadOnlyList<MediaStream> streams, ManagedConnection db) { const int Limit = 10; var startIndex = 0; @@ -5721,7 +5721,7 @@ AND Type = @InternalPersonType)"); cmdText += " order by AttachmentIndex ASC"; var list = new List<MediaAttachment>(); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) using (var statement = PrepareStatement(connection, cmdText)) { statement.TryBind("@ItemId", query.ItemId); @@ -5771,7 +5771,7 @@ AND Type = @InternalPersonType)"); private void InsertMediaAttachments( Guid id, IReadOnlyList<MediaAttachment> attachments, - SqliteConnection db, + ManagedConnection db, CancellationToken cancellationToken) { const int InsertAtOnce = 10; diff --git a/Emby.Server.Implementations/Data/SqliteUserDataRepository.cs b/Emby.Server.Implementations/Data/SqliteUserDataRepository.cs index 20359e4ad..634eaf85e 100644 --- a/Emby.Server.Implementations/Data/SqliteUserDataRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteUserDataRepository.cs @@ -86,7 +86,7 @@ namespace Emby.Server.Implementations.Data } } - private void ImportUserIds(SqliteConnection db, IEnumerable<User> users) + private void ImportUserIds(ManagedConnection db, IEnumerable<User> users) { var userIdsWithUserData = GetAllUserIdsWithUserData(db); @@ -107,7 +107,7 @@ namespace Emby.Server.Implementations.Data } } - private List<Guid> GetAllUserIdsWithUserData(SqliteConnection db) + private List<Guid> GetAllUserIdsWithUserData(ManagedConnection db) { var list = new List<Guid>(); @@ -176,7 +176,7 @@ namespace Emby.Server.Implementations.Data } } - private static void SaveUserData(SqliteConnection db, long internalUserId, string key, UserItemData userData) + private static void SaveUserData(ManagedConnection db, long internalUserId, string key, UserItemData userData) { using (var statement = db.PrepareStatement("replace into UserDatas (key, userId, rating,played,playCount,isFavorite,playbackPositionTicks,lastPlayedDate,AudioStreamIndex,SubtitleStreamIndex) values (@key, @userId, @rating,@played,@playCount,@isFavorite,@playbackPositionTicks,@lastPlayedDate,@AudioStreamIndex,@SubtitleStreamIndex)")) { @@ -267,7 +267,7 @@ namespace Emby.Server.Implementations.Data ArgumentException.ThrowIfNullOrEmpty(key); - using (var connection = GetConnection()) + using (var connection = GetConnection(true)) { using (var statement = connection.PrepareStatement("select key,userid,rating,played,playCount,isFavorite,playbackPositionTicks,lastPlayedDate,AudioStreamIndex,SubtitleStreamIndex from UserDatas where key =@Key and userId=@UserId")) { diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index e66f2496a..953fe19e0 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1029,7 +1029,7 @@ namespace Emby.Server.Implementations.Library } } - private async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) + public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) { await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); diff --git a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs index 733ae2d1a..abf2d0115 100644 --- a/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs +++ b/Emby.Server.Implementations/Library/Resolvers/TV/SeasonResolver.cs @@ -55,7 +55,7 @@ namespace Emby.Server.Implementations.Library.Resolvers.TV IndexNumber = seasonParserResult.SeasonNumber, SeriesId = series.Id, SeriesName = series.Name, - Path = seasonParserResult.IsSeasonFolder ? path : args.Parent.Path + Path = seasonParserResult.IsSeasonFolder ? path : null }; if (!season.IndexNumber.HasValue || !seasonParserResult.IsSeasonFolder) diff --git a/Emby.Server.Implementations/TV/TVSeriesManager.cs b/Emby.Server.Implementations/TV/TVSeriesManager.cs index 34c9e86f2..c1a615666 100644 --- a/Emby.Server.Implementations/TV/TVSeriesManager.cs +++ b/Emby.Server.Implementations/TV/TVSeriesManager.cs @@ -91,7 +91,7 @@ namespace Emby.Server.Implementations.TV } string? presentationUniqueKey = null; - int? limit = null; + int? limit = request.Limit; if (!request.SeriesId.IsNullOrEmpty()) { if (_libraryManager.GetItemById(request.SeriesId.Value) is Series series) diff --git a/Jellyfin.Api/Controllers/LibraryStructureController.cs b/Jellyfin.Api/Controllers/LibraryStructureController.cs index bff578feb..b6de67e88 100644 --- a/Jellyfin.Api/Controllers/LibraryStructureController.cs +++ b/Jellyfin.Api/Controllers/LibraryStructureController.cs @@ -179,7 +179,21 @@ public class LibraryStructureController : BaseJellyfinApiController // No need to start if scanning the library because it will handle it if (refreshLibrary) { - await _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None).ConfigureAwait(false); + await _libraryManager.ValidateTopLibraryFolders(CancellationToken.None, true).ConfigureAwait(false); + var newLib = _libraryManager.GetUserRootFolder().Children.FirstOrDefault(f => f.Path.Equals(newPath, StringComparison.OrdinalIgnoreCase)); + if (newLib is CollectionFolder folder) + { + foreach (var child in folder.GetPhysicalFolders()) + { + await child.RefreshMetadata(CancellationToken.None).ConfigureAwait(false); + await child.ValidateChildren(new Progress<double>(), CancellationToken.None).ConfigureAwait(false); + } + } + else + { + // We don't know if this one can be validated individually, trigger a new validation + await _libraryManager.ValidateMediaLibrary(new Progress<double>(), CancellationToken.None).ConfigureAwait(false); + } } else { diff --git a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs index 74f7e9c3e..a20253369 100644 --- a/Jellyfin.Server/Migrations/Routines/FixAudioData.cs +++ b/Jellyfin.Server/Migrations/Routines/FixAudioData.cs @@ -55,8 +55,9 @@ namespace Jellyfin.Server.Migrations.Routines { try { + _logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); File.Copy(dbPath, bakPath); - _logger.LogInformation("Library database backed up to {BackupPath}", bakPath); + _logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); break; } catch (Exception ex) @@ -80,7 +81,7 @@ namespace Jellyfin.Server.Migrations.Routines { IncludeItemTypes = [BaseItemKind.Audio], StartIndex = startIndex, - Limit = 100, + Limit = 5000, SkipDeserialization = true }) .Cast<Audio>() @@ -97,7 +98,8 @@ namespace Jellyfin.Server.Migrations.Routines } _itemRepository.SaveItems(results, CancellationToken.None); - startIndex += 100; + startIndex += results.Count; + _logger.LogInformation("Backfilled data for {UpdatedRecords} of {TotalRecords} audio records", startIndex, records); } } } diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 0f2ec1f8f..fcb45e7e5 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -364,7 +364,7 @@ namespace MediaBrowser.Controller.Entities if (IsFileProtocol) { - IEnumerable<BaseItem> nonCachedChildren; + IEnumerable<BaseItem> nonCachedChildren = []; try { @@ -373,7 +373,6 @@ namespace MediaBrowser.Controller.Entities catch (Exception ex) { Logger.LogError(ex, "Error retrieving children folder"); - return; } progress.Report(ProgressHelpers.RetrievedChildren); diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 37703ceee..b802b7e6e 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -149,6 +149,14 @@ namespace MediaBrowser.Controller.Library /// <returns>Task.</returns> Task ValidateMediaLibrary(IProgress<double> progress, CancellationToken cancellationToken); + /// <summary> + /// Reloads the root media folder. + /// </summary> + /// <param name="cancellationToken">The cancellation token.</param> + /// <param name="removeRoot">Is remove the library itself allowed.</param> + /// <returns>Task.</returns> + Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false); + Task UpdateImagesAsync(BaseItem item, bool forceUpdate = false); /// <summary> diff --git a/MediaBrowser.Controller/Providers/DirectoryService.cs b/MediaBrowser.Controller/Providers/DirectoryService.cs index 7fe2f64af..56b07ebae 100644 --- a/MediaBrowser.Controller/Providers/DirectoryService.cs +++ b/MediaBrowser.Controller/Providers/DirectoryService.cs @@ -28,6 +28,22 @@ namespace MediaBrowser.Controller.Providers return _cache.GetOrAdd(path, static (p, fileSystem) => fileSystem.GetFileSystemEntries(p).ToArray(), _fileSystem); } + public List<FileSystemMetadata> GetDirectories(string path) + { + var list = new List<FileSystemMetadata>(); + var items = GetFileSystemEntries(path); + for (var i = 0; i < items.Length; i++) + { + var item = items[i]; + if (item.IsDirectory) + { + list.Add(item); + } + } + + return list; + } + public List<FileSystemMetadata> GetFiles(string path) { var list = new List<FileSystemMetadata>(); diff --git a/MediaBrowser.Controller/Providers/IDirectoryService.cs b/MediaBrowser.Controller/Providers/IDirectoryService.cs index 6d7550ab5..a3c06cde5 100644 --- a/MediaBrowser.Controller/Providers/IDirectoryService.cs +++ b/MediaBrowser.Controller/Providers/IDirectoryService.cs @@ -9,6 +9,8 @@ namespace MediaBrowser.Controller.Providers { FileSystemMetadata[] GetFileSystemEntries(string path); + List<FileSystemMetadata> GetDirectories(string path); + List<FileSystemMetadata> GetFiles(string path); FileSystemMetadata? GetFile(string path); diff --git a/MediaBrowser.Controller/Providers/IProviderManager.cs b/MediaBrowser.Controller/Providers/IProviderManager.cs index eb5069b06..b52f16edc 100644 --- a/MediaBrowser.Controller/Providers/IProviderManager.cs +++ b/MediaBrowser.Controller/Providers/IProviderManager.cs @@ -141,6 +141,14 @@ namespace MediaBrowser.Controller.Providers where T : BaseItem; /// <summary> + /// Gets the metadata savers for the provided item. + /// </summary> + /// <param name="item">The item.</param> + /// <param name="libraryOptions">The library options.</param> + /// <returns>The metadata savers.</returns> + IEnumerable<IMetadataSaver> GetMetadataSavers(BaseItem item, LibraryOptions libraryOptions); + + /// <summary> /// Gets all metadata plugins. /// </summary> /// <returns>IEnumerable{MetadataPlugin}.</returns> diff --git a/MediaBrowser.LocalMetadata/Images/EpisodeLocalImageProvider.cs b/MediaBrowser.LocalMetadata/Images/EpisodeLocalImageProvider.cs index a8e2946f1..6a5e3bf04 100644 --- a/MediaBrowser.LocalMetadata/Images/EpisodeLocalImageProvider.cs +++ b/MediaBrowser.LocalMetadata/Images/EpisodeLocalImageProvider.cs @@ -38,19 +38,28 @@ namespace MediaBrowser.LocalMetadata.Images } var parentPathFiles = directoryService.GetFiles(parentPath); + var nameWithoutExtension = Path.GetFileNameWithoutExtension(item.Path.AsSpan()).ToString(); - var nameWithoutExtension = Path.GetFileNameWithoutExtension(item.Path.AsSpan()); + var thumbName = string.Concat(nameWithoutExtension, "-thumb"); + var images = GetImageFilesFromFolder(thumbName, parentPathFiles); - return GetFilesFromParentFolder(nameWithoutExtension, parentPathFiles); + var metadataSubPath = directoryService.GetDirectories(parentPath).Where(d => d.Name.EndsWith("metadata", StringComparison.OrdinalIgnoreCase)).ToList(); + foreach (var path in metadataSubPath) + { + var files = directoryService.GetFiles(path.FullName); + images.AddRange(GetImageFilesFromFolder(nameWithoutExtension, files)); + } + + return images; } - private List<LocalImageInfo> GetFilesFromParentFolder(ReadOnlySpan<char> filenameWithoutExtension, List<FileSystemMetadata> parentPathFiles) + private List<LocalImageInfo> GetImageFilesFromFolder(ReadOnlySpan<char> filenameWithoutExtension, List<FileSystemMetadata> filePaths) { var thumbName = string.Concat(filenameWithoutExtension, "-thumb"); var list = new List<LocalImageInfo>(1); - foreach (var i in parentPathFiles) + foreach (var i in filePaths) { if (i.IsDirectory) { diff --git a/MediaBrowser.Providers/Manager/ImageSaver.cs b/MediaBrowser.Providers/Manager/ImageSaver.cs index d82716831..e0677aa9f 100644 --- a/MediaBrowser.Providers/Manager/ImageSaver.cs +++ b/MediaBrowser.Providers/Manager/ImageSaver.cs @@ -100,8 +100,8 @@ namespace MediaBrowser.Providers.Manager { saveLocally = false; - // If season is virtual under a physical series, save locally if using compatible convention - if (item is Season season && _config.Configuration.ImageSavingConvention == ImageSavingConvention.Compatible) + // If season is virtual under a physical series, save locally + if (item is Season season) { var series = season.Series; @@ -126,7 +126,11 @@ namespace MediaBrowser.Providers.Manager var paths = GetSavePaths(item, type, imageIndex, mimeType, saveLocally); - var retryPaths = GetSavePaths(item, type, imageIndex, mimeType, false); + string[] retryPaths = []; + if (saveLocally) + { + retryPaths = GetSavePaths(item, type, imageIndex, mimeType, false); + } // If there are more than one output paths, the stream will need to be seekable if (paths.Length > 1 && !source.CanSeek) @@ -183,6 +187,13 @@ namespace MediaBrowser.Providers.Manager try { _fileSystem.DeleteFile(currentPath); + + // Remove containing directory if empty + var folder = Path.GetDirectoryName(currentPath); + if (!_fileSystem.GetFiles(folder).Any()) + { + Directory.Delete(folder); + } } catch (FileNotFoundException) { @@ -374,6 +385,45 @@ namespace MediaBrowser.Providers.Manager throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "Unable to determine image file extension from mime type {0}", mimeType)); } + if (string.Equals(extension, ".jpeg", StringComparison.OrdinalIgnoreCase)) + { + extension = ".jpg"; + } + + extension = extension.ToLowerInvariant(); + + if (type == ImageType.Primary && saveLocally) + { + if (season is not null && season.IndexNumber.HasValue) + { + var seriesFolder = season.SeriesPath; + + var seasonMarker = season.IndexNumber.Value == 0 + ? "-specials" + : season.IndexNumber.Value.ToString("00", CultureInfo.InvariantCulture); + + var imageFilename = "season" + seasonMarker + "-poster" + extension; + + return Path.Combine(seriesFolder, imageFilename); + } + } + + if (type == ImageType.Backdrop && saveLocally) + { + if (season is not null && season.IndexNumber.HasValue) + { + var seriesFolder = season.SeriesPath; + + var seasonMarker = season.IndexNumber.Value == 0 + ? "-specials" + : season.IndexNumber.Value.ToString("00", CultureInfo.InvariantCulture); + + var imageFilename = "season" + seasonMarker + "-fanart" + extension; + + return Path.Combine(seriesFolder, imageFilename); + } + } + if (type == ImageType.Thumb && saveLocally) { if (season is not null && season.IndexNumber.HasValue) @@ -447,20 +497,12 @@ namespace MediaBrowser.Providers.Manager break; } - if (string.Equals(extension, ".jpeg", StringComparison.OrdinalIgnoreCase)) - { - extension = ".jpg"; - } - - extension = extension.ToLowerInvariant(); - string path = null; - if (saveLocally) { if (type == ImageType.Primary && item is Episode) { - path = Path.Combine(Path.GetDirectoryName(item.Path), "metadata", filename + extension); + path = Path.Combine(Path.GetDirectoryName(item.Path), filename + "-thumb" + extension); } else if (item.IsInMixedFolder) { diff --git a/MediaBrowser.Providers/Manager/ItemImageProvider.cs b/MediaBrowser.Providers/Manager/ItemImageProvider.cs index 1a5dbd7a5..bee420d95 100644 --- a/MediaBrowser.Providers/Manager/ItemImageProvider.cs +++ b/MediaBrowser.Providers/Manager/ItemImageProvider.cs @@ -371,12 +371,21 @@ namespace MediaBrowser.Providers.Manager } catch (FileNotFoundException) { - // nothing to do, already gone + // Nothing to do, already gone } catch (UnauthorizedAccessException ex) { _logger.LogWarning(ex, "Unable to delete {Image}", image.Path); } + finally + { + // Always remove empty parent folder + var folder = Path.GetDirectoryName(image.Path); + if (Directory.Exists(folder) && !_fileSystem.GetFiles(folder).Any()) + { + Directory.Delete(folder); + } + } } } @@ -419,7 +428,8 @@ namespace MediaBrowser.Providers.Manager var type = _singularImages[i]; var image = GetFirstLocalImageInfoByType(images, type); - if (image is not null) + // Only use local images if we are not replacing and saving + if (image is not null && !(item.IsSaveLocalMetadataEnabled() && refreshOptions.ReplaceAllImages)) { var currentImage = item.GetImageInfo(type, 0); // if image file is stored with media, don't replace that later diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 0a98967da..61a4d7586 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -154,7 +154,8 @@ namespace MediaBrowser.Providers.Manager id.IsAutomated = refreshOptions.IsAutomated; - var result = await RefreshWithProviders(metadataResult, id, refreshOptions, providers, ImageProvider, cancellationToken).ConfigureAwait(false); + var hasMetadataSavers = ProviderManager.GetMetadataSavers(item, libraryOptions).Any(); + var result = await RefreshWithProviders(metadataResult, id, refreshOptions, providers, ImageProvider, hasMetadataSavers, cancellationToken).ConfigureAwait(false); updateType |= result.UpdateType; if (result.Failures > 0) @@ -639,6 +640,7 @@ namespace MediaBrowser.Providers.Manager MetadataRefreshOptions options, ICollection<IMetadataProvider> providers, ItemImageProvider imageService, + bool isSavingMetadata, CancellationToken cancellationToken) { var refreshResult = new RefreshResult @@ -669,69 +671,74 @@ namespace MediaBrowser.Providers.Manager temp.Item.Id = item.Id; var foundImageTypes = new List<ImageType>(); - foreach (var provider in providers.OfType<ILocalMetadataProvider<TItemType>>()) - { - var providerName = provider.GetType().Name; - Logger.LogDebug("Running {Provider} for {Item}", providerName, logName); - var itemInfo = new ItemInfo(item); - - try + // Do not execute local providers if we are identifying or replacing with local metadata saving enabled + if (options.SearchResult is null && !(isSavingMetadata && options.ReplaceAllMetadata)) + { + foreach (var provider in providers.OfType<ILocalMetadataProvider<TItemType>>()) { - var localItem = await provider.GetMetadata(itemInfo, options.DirectoryService, cancellationToken).ConfigureAwait(false); + var providerName = provider.GetType().Name; + Logger.LogDebug("Running {Provider} for {Item}", providerName, logName); + + var itemInfo = new ItemInfo(item); - if (localItem.HasMetadata) + try { - foreach (var remoteImage in localItem.RemoteImages) + var localItem = await provider.GetMetadata(itemInfo, options.DirectoryService, cancellationToken).ConfigureAwait(false); + + if (localItem.HasMetadata) { - try + foreach (var remoteImage in localItem.RemoteImages) { - if (item.ImageInfos.Any(x => x.Type == remoteImage.Type) - && !options.IsReplacingImage(remoteImage.Type)) + try { - continue; - } + if (item.ImageInfos.Any(x => x.Type == remoteImage.Type) + && !options.IsReplacingImage(remoteImage.Type)) + { + continue; + } - await ProviderManager.SaveImage(item, remoteImage.Url, remoteImage.Type, null, cancellationToken).ConfigureAwait(false); - refreshResult.UpdateType |= ItemUpdateType.ImageUpdate; + await ProviderManager.SaveImage(item, remoteImage.Url, remoteImage.Type, null, cancellationToken).ConfigureAwait(false); + refreshResult.UpdateType |= ItemUpdateType.ImageUpdate; - // remember imagetype that has just been downloaded - foundImageTypes.Add(remoteImage.Type); + // remember imagetype that has just been downloaded + foundImageTypes.Add(remoteImage.Type); + } + catch (HttpRequestException ex) + { + Logger.LogError(ex, "Could not save {ImageType} image: {Url}", Enum.GetName(remoteImage.Type), remoteImage.Url); + } } - catch (HttpRequestException ex) + + if (foundImageTypes.Count > 0) { - Logger.LogError(ex, "Could not save {ImageType} image: {Url}", Enum.GetName(remoteImage.Type), remoteImage.Url); + imageService.UpdateReplaceImages(options, foundImageTypes); } - } - if (foundImageTypes.Count > 0) - { - imageService.UpdateReplaceImages(options, foundImageTypes); - } + if (imageService.MergeImages(item, localItem.Images, options)) + { + refreshResult.UpdateType |= ItemUpdateType.ImageUpdate; + } - if (imageService.MergeImages(item, localItem.Images, options)) - { - refreshResult.UpdateType |= ItemUpdateType.ImageUpdate; - } + MergeData(localItem, temp, Array.Empty<MetadataField>(), false, true); + refreshResult.UpdateType |= ItemUpdateType.MetadataImport; - MergeData(localItem, temp, Array.Empty<MetadataField>(), false, true); - refreshResult.UpdateType |= ItemUpdateType.MetadataImport; + break; + } - break; + Logger.LogDebug("{Provider} returned no metadata for {Item}", providerName, logName); } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Logger.LogError(ex, "Error in {Provider}", provider.Name); - Logger.LogDebug("{Provider} returned no metadata for {Item}", providerName, logName); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - Logger.LogError(ex, "Error in {Provider}", provider.Name); - - // If a local provider fails, consider that a failure - refreshResult.ErrorMessage = ex.Message; + // If a local provider fails, consider that a failure + refreshResult.ErrorMessage = ex.Message; + } } } @@ -763,7 +770,7 @@ namespace MediaBrowser.Providers.Manager else { var shouldReplace = options.MetadataRefreshMode > MetadataRefreshMode.ValidationOnly || options.ReplaceAllMetadata; - MergeData(temp, metadata, item.LockedFields, shouldReplace, false); + MergeData(temp, metadata, item.LockedFields, shouldReplace, true); } } } @@ -1050,7 +1057,7 @@ namespace MediaBrowser.Providers.Manager } else { - target.Tags = target.ProductionLocations.Concat(source.ProductionLocations).Distinct().ToArray(); + target.ProductionLocations = target.ProductionLocations.Concat(source.ProductionLocations).Distinct().ToArray(); } } @@ -1080,7 +1087,7 @@ namespace MediaBrowser.Providers.Manager } else { - target.RemoteTrailers = target.RemoteTrailers.Concat(source.RemoteTrailers).Distinct().ToArray(); + target.RemoteTrailers = target.RemoteTrailers.Concat(source.RemoteTrailers).DistinctBy(t => t.Url).ToArray(); } MergeAlbumArtist(source, target, replaceData); diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index 275f4028d..f2ca99da6 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -418,6 +418,12 @@ namespace MediaBrowser.Providers.Manager return GetMetadataProvidersInternal<T>(item, libraryOptions, globalMetadataOptions, false, false); } + /// <inheritdoc /> + public IEnumerable<IMetadataSaver> GetMetadataSavers(BaseItem item, LibraryOptions libraryOptions) + { + return _savers.Where(i => IsSaverEnabledForItem(i, item, libraryOptions, ItemUpdateType.MetadataEdit, false)); + } + private IEnumerable<IMetadataProvider<T>> GetMetadataProvidersInternal<T>(BaseItem item, LibraryOptions libraryOptions, MetadataOptions globalMetadataOptions, bool includeDisabled, bool forceEnableInternetMetadata) where T : BaseItem { diff --git a/MediaBrowser.Providers/TV/SeriesMetadataService.cs b/MediaBrowser.Providers/TV/SeriesMetadataService.cs index 41c6bd8f2..2389bce57 100644 --- a/MediaBrowser.Providers/TV/SeriesMetadataService.cs +++ b/MediaBrowser.Providers/TV/SeriesMetadataService.cs @@ -119,7 +119,8 @@ namespace MediaBrowser.Providers.TV virtualSeason, new DeleteOptions { - DeleteFileLocation = true + // Internal metadata paths are removed regardless of this. + DeleteFileLocation = false }, false); } @@ -176,7 +177,8 @@ namespace MediaBrowser.Providers.TV episode, new DeleteOptions { - DeleteFileLocation = true + // Internal metadata paths are removed regardless of this. + DeleteFileLocation = false }, false); } diff --git a/MediaBrowser.XbmcMetadata/Savers/BaseNfoSaver.cs b/MediaBrowser.XbmcMetadata/Savers/BaseNfoSaver.cs index b25cfc83f..a547779de 100644 --- a/MediaBrowser.XbmcMetadata/Savers/BaseNfoSaver.cs +++ b/MediaBrowser.XbmcMetadata/Savers/BaseNfoSaver.cs @@ -825,7 +825,7 @@ namespace MediaBrowser.XbmcMetadata.Savers private string GetOutputTrailerUrl(string url) { // This is what xbmc expects - return url.Replace(YouTubeWatchUrl, "plugin://plugin.video.youtube/?action=play_video&videoid=", StringComparison.OrdinalIgnoreCase); + return url.Replace(YouTubeWatchUrl, "plugin://plugin.video.youtube/play/?video_id=", StringComparison.OrdinalIgnoreCase); } private void AddImages(BaseItem item, XmlWriter writer, ILibraryManager libraryManager) diff --git a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs index 8fa22fad9..bc344d87e 100644 --- a/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs +++ b/MediaBrowser.XbmcMetadata/Savers/MovieNfoSaver.cs @@ -45,27 +45,24 @@ namespace MediaBrowser.XbmcMetadata.Savers internal static IEnumerable<string> GetMovieSavePaths(ItemInfo item) { + var path = item.ContainingFolderPath; if (item.VideoType == VideoType.Dvd && !item.IsPlaceHolder) { - var path = item.ContainingFolderPath; - yield return Path.Combine(path, "VIDEO_TS", "VIDEO_TS.nfo"); } - if (!item.IsPlaceHolder && (item.VideoType == VideoType.Dvd || item.VideoType == VideoType.BluRay)) + // only allow movie object to read movie.nfo, not owned videos (which will be itemtype video, not movie) + if (!item.IsInMixedFolder && item.ItemType == typeof(Movie)) { - var path = item.ContainingFolderPath; + yield return Path.Combine(path, "movie.nfo"); + } + if (!item.IsPlaceHolder && (item.VideoType == VideoType.Dvd || item.VideoType == VideoType.BluRay)) + { yield return Path.Combine(path, Path.GetFileName(path) + ".nfo"); } else { - // only allow movie object to read movie.nfo, not owned videos (which will be itemtype video, not movie) - if (!item.IsInMixedFolder && item.ItemType == typeof(Movie)) - { - yield return Path.Combine(item.ContainingFolderPath, "movie.nfo"); - } - yield return Path.ChangeExtension(item.Path, ".nfo"); } } @@ -16,9 +16,6 @@ <a href="https://translate.jellyfin.org/projects/jellyfin/jellyfin-core/?utm_source=widget"> <img alt="Translation Status" src="https://translate.jellyfin.org/widgets/jellyfin/-/jellyfin-core/svg-badge.svg"/> </a> -<a href="https://dev.azure.com/jellyfin-project/jellyfin/_build?definitionId=29"> -<img alt="Azure Builds" src="https://dev.azure.com/jellyfin-project/jellyfin/_apis/build/status/Jellyfin%20Server"/> -</a> <a href="https://hub.docker.com/r/jellyfin/jellyfin"> <img alt="Docker Pull Count" src="https://img.shields.io/docker/pulls/jellyfin/jellyfin.svg"/> </a> @@ -30,10 +27,7 @@ <img alt="Submit Feature Requests" src="https://img.shields.io/badge/fider-vote%20on%20features-success.svg"/> </a> <a href="https://matrix.to/#/#jellyfinorg:matrix.org"> -<img alt="Chat on Matrix" src="https://img.shields.io/matrix/jellyfin:matrix.org.svg?logo=matrix"/> -</a> -<a href="https://www.reddit.com/r/jellyfin"> -<img alt="Join our Subreddit" src="https://img.shields.io/badge/reddit-r%2Fjellyfin-%23FF5700.svg"/> +<img alt="Chat on Matrix" src="https://img.shields.io/matrix/jellyfinorg:matrix.org.svg?logo=matrix"/> </a> <a href="https://github.com/jellyfin/jellyfin/releases.atom"> <img alt="Release RSS Feed" src="https://img.shields.io/badge/rss-releases-ffa500?logo=rss" /> diff --git a/tests/Jellyfin.Providers.Tests/Manager/ItemImageProviderTests.cs b/tests/Jellyfin.Providers.Tests/Manager/ItemImageProviderTests.cs index be5a401b1..d6284e4a1 100644 --- a/tests/Jellyfin.Providers.Tests/Manager/ItemImageProviderTests.cs +++ b/tests/Jellyfin.Providers.Tests/Manager/ItemImageProviderTests.cs @@ -575,18 +575,22 @@ namespace Jellyfin.Providers.Tests.Manager // Has to exist for querying DateModified time on file, results stored but not checked so not populating BaseItem.FileSystem ??= Mock.Of<IFileSystem>(); - var item = new Video(); + var item = new Mock<Video> + { + CallBase = true + }; + item.Setup(m => m.IsSaveLocalMetadataEnabled()).Returns(false); var path = validPaths ? _testDataImagePath.Format : "invalid path {0}"; for (int i = 0; i < count; i++) { - item.SetImagePath(type, i, new FileSystemMetadata + item.Object.SetImagePath(type, i, new FileSystemMetadata { FullName = string.Format(CultureInfo.InvariantCulture, path, i), }); } - return item; + return item.Object; } private static ILocalImageProvider GetImageProvider(ImageType type, int count, bool validPaths) diff --git a/tests/Jellyfin.XbmcMetadata.Tests/Location/MovieNfoLocationTests.cs b/tests/Jellyfin.XbmcMetadata.Tests/Location/MovieNfoLocationTests.cs index 8019e0ab3..2f05c4ea2 100644 --- a/tests/Jellyfin.XbmcMetadata.Tests/Location/MovieNfoLocationTests.cs +++ b/tests/Jellyfin.XbmcMetadata.Tests/Location/MovieNfoLocationTests.cs @@ -47,6 +47,7 @@ namespace Jellyfin.XbmcMetadata.Tests.Location var movie = new Movie() { Path = "/media/movies/Avengers Endgame", VideoType = VideoType.Dvd }; var path1 = "/media/movies/Avengers Endgame/Avengers Endgame.nfo"; var path2 = "/media/movies/Avengers Endgame/VIDEO_TS/VIDEO_TS.nfo"; + var path3 = "/media/movies/Avengers Endgame/movie.nfo"; // uses ContainingFolderPath which uses Operating system specific paths if (OperatingSystem.IsWindows()) @@ -54,12 +55,14 @@ namespace Jellyfin.XbmcMetadata.Tests.Location movie.Path = movie.Path.Replace('/', '\\'); path1 = path1.Replace('/', '\\'); path2 = path2.Replace('/', '\\'); + path3 = path3.Replace('/', '\\'); } var paths = MovieNfoSaver.GetMovieSavePaths(new ItemInfo(movie)).ToArray(); - Assert.Equal(2, paths.Length); + Assert.Equal(3, paths.Length); Assert.Contains(path1, paths); Assert.Contains(path2, paths); + Assert.Contains(path3, paths); } } } |