diff options
Diffstat (limited to 'Emby.Server.Implementations')
4 files changed, 258 insertions, 42 deletions
diff --git a/Emby.Server.Implementations/ApplicationHost.cs b/Emby.Server.Implementations/ApplicationHost.cs index e8cab6ea8c..b7aa2f3d06 100644 --- a/Emby.Server.Implementations/ApplicationHost.cs +++ b/Emby.Server.Implementations/ApplicationHost.cs @@ -536,6 +536,7 @@ namespace Emby.Server.Implementations serviceCollection.AddSingleton<IMusicManager, MusicManager>(); serviceCollection.AddSingleton<ILibraryMonitor, LibraryMonitor>(); + serviceCollection.AddSingleton<DotIgnoreIgnoreRule>(); serviceCollection.AddSingleton<ISearchEngine, SearchEngine>(); diff --git a/Emby.Server.Implementations/IO/LibraryMonitor.cs b/Emby.Server.Implementations/IO/LibraryMonitor.cs index 23bd5cf200..1bf0f8c76c 100644 --- a/Emby.Server.Implementations/IO/LibraryMonitor.cs +++ b/Emby.Server.Implementations/IO/LibraryMonitor.cs @@ -21,6 +21,7 @@ namespace Emby.Server.Implementations.IO private readonly ILibraryManager _libraryManager; private readonly IServerConfigurationManager _configurationManager; private readonly IFileSystem _fileSystem; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// <summary> /// The file system watchers. @@ -47,17 +48,20 @@ namespace Emby.Server.Implementations.IO /// <param name="configurationManager">The configuration manager.</param> /// <param name="fileSystem">The filesystem.</param> /// <param name="appLifetime">The <see cref="IHostApplicationLifetime"/>.</param> + /// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param> public LibraryMonitor( ILogger<LibraryMonitor> logger, ILibraryManager libraryManager, IServerConfigurationManager configurationManager, IFileSystem fileSystem, - IHostApplicationLifetime appLifetime) + IHostApplicationLifetime appLifetime, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _libraryManager = libraryManager; _logger = logger; _configurationManager = configurationManager; _fileSystem = fileSystem; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; appLifetime.ApplicationStarted.Register(Start); appLifetime.ApplicationStopping.Register(Stop); @@ -354,7 +358,7 @@ namespace Emby.Server.Implementations.IO } var fileInfo = _fileSystem.GetFileSystemInfo(path); - if (DotIgnoreIgnoreRule.IsIgnored(fileInfo, null)) + if (_dotIgnoreIgnoreRule.ShouldIgnore(fileInfo, null)) { return; } diff --git a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs index ef5d24c70f..f073fc0bca 100644 --- a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs +++ b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Text.RegularExpressions; +using BitFaster.Caching.Lru; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Resolvers; @@ -15,22 +16,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule { private static readonly bool IsWindows = OperatingSystem.IsWindows(); - private static FileInfo? FindIgnoreFile(DirectoryInfo directory) - { - for (var current = directory; current is not null; current = current.Parent) - { - var ignorePath = Path.Join(current.FullName, ".ignore"); - if (File.Exists(ignorePath)) - { - return new FileInfo(ignorePath); - } - } + private readonly FastConcurrentLru<string, IgnoreFileCacheEntry> _directoryCache; + private readonly FastConcurrentLru<string, ParsedIgnoreCacheEntry> _rulesCache; - return null; + /// <summary> + /// Initializes a new instance of the <see cref="DotIgnoreIgnoreRule"/> class. + /// </summary> + public DotIgnoreIgnoreRule() + { + var cacheSize = Math.Max(100, Environment.ProcessorCount * 100); + _directoryCache = new FastConcurrentLru<string, IgnoreFileCacheEntry>( + Environment.ProcessorCount, + cacheSize, + StringComparer.Ordinal); + _rulesCache = new FastConcurrentLru<string, ParsedIgnoreCacheEntry>( + Environment.ProcessorCount, + Math.Max(32, cacheSize / 4), + StringComparer.Ordinal); } /// <inheritdoc /> - public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent); + public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent); + + /// <summary> + /// Clears the directory lookup cache. The parsed rules cache is not cleared + /// as it validates file modification time on each access. + /// </summary> + public void ClearDirectoryCache() + { + _directoryCache.Clear(); + } /// <summary> /// Checks whether or not the file is ignored. @@ -38,40 +53,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule /// <param name="fileInfo">The file information.</param> /// <param name="parent">The parent BaseItem.</param> /// <returns>True if the file should be ignored.</returns> - public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent) + public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent) { var searchDirectory = fileInfo.IsDirectory - ? new DirectoryInfo(fileInfo.FullName) - : new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty); + ? fileInfo.FullName + : Path.GetDirectoryName(fileInfo.FullName); - if (string.IsNullOrEmpty(searchDirectory.FullName)) + if (string.IsNullOrEmpty(searchDirectory)) { return false; } - var ignoreFile = FindIgnoreFile(searchDirectory); - if (ignoreFile is null) + var ignoreFilePath = FindIgnoreFileCached(searchDirectory); + if (ignoreFilePath is null) { return false; } - // Fast path in case the ignore files isn't a symlink and is empty - if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0) + var parsedEntry = GetParsedRules(ignoreFilePath); + if (parsedEntry is null) { - // Ignore directory if we just have the file - return true; + // File was deleted after we cached the path - clear the directory cache entry and return false + _directoryCache.TryRemove(searchDirectory, out _); + return false; } - var content = GetFileContent(ignoreFile); - return string.IsNullOrWhiteSpace(content) - || CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory); - } + // Empty file means ignore everything + if (parsedEntry.IsEmpty) + { + return true; + } - private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory) - { - // If file has content, base ignoring off the content .gitignore-style rules - var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - return CheckIgnoreRules(path, rules, isDirectory); + return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory)); } /// <summary> @@ -117,8 +130,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return true; } - // Mitigate the problem of the Ignore library not handling Windows paths correctly. - // See https://github.com/jellyfin/jellyfin/issues/15484 + // Mitigate the problem of the Ignore library not handling Windows paths correctly. + // See https://github.com/jellyfin/jellyfin/issues/15484 var pathToCheck = normalizePath ? path.NormalizePath('/') : path; // Add trailing slash for directories to match "folder/" @@ -130,11 +143,194 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return ignore.IsIgnored(pathToCheck); } - private static string GetFileContent(FileInfo ignoreFile) + private string? FindIgnoreFileCached(string directory) + { + // Check if we have a cached result for this directory + if (_directoryCache.TryGet(directory, out var cached)) + { + return cached.IgnoreFilePath; + } + + // Walk up the directory tree to find .ignore file + var current = directory; + var checkedDirs = new System.Collections.Generic.List<string> { directory }; + + while (!string.IsNullOrEmpty(current)) + { + // Check if this intermediate directory is cached + if (current != directory && _directoryCache.TryGet(current, out var parentCached)) + { + // Cache the result for all directories we checked + var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFilePath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return parentCached.IgnoreFilePath; + } + + var ignorePath = Path.Join(current, ".ignore"); + if (File.Exists(ignorePath)) + { + // Cache for all directories we checked + var entry = new IgnoreFileCacheEntry(ignorePath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return ignorePath; + } + + var parent = Path.GetDirectoryName(current); + if (parent == current || string.IsNullOrEmpty(parent)) + { + break; + } + + current = parent; + checkedDirs.Add(current); + } + + // No .ignore file found - cache null result for all directories + var nullEntry = new IgnoreFileCacheEntry(null); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, nullEntry); + } + + return null; + } + + private ParsedIgnoreCacheEntry? GetParsedRules(string ignoreFilePath) { - ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; - return ignoreFile.Exists - ? File.ReadAllText(ignoreFile.FullName) - : string.Empty; + FileInfo fileInfo; + try + { + fileInfo = new FileInfo(ignoreFilePath); + if (!fileInfo.Exists) + { + _rulesCache.TryRemove(ignoreFilePath, out _); + return null; + } + } + catch + { + _rulesCache.TryRemove(ignoreFilePath, out _); + return null; + } + + var lastModified = fileInfo.LastWriteTimeUtc; + var fileLength = fileInfo.Length; + + // Check cache + if (_rulesCache.TryGet(ignoreFilePath, out var cached)) + { + if (cached.FileLastModified == lastModified && cached.FileLength == fileLength) + { + return cached; + } + + // Stale - need to reparse + _rulesCache.TryRemove(ignoreFilePath, out _); + } + + // Parse the file + var parsedEntry = ParseIgnoreFile(fileInfo, lastModified, fileLength); + _rulesCache.AddOrUpdate(ignoreFilePath, parsedEntry); + return parsedEntry; + } + + private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength) + { + if (ignoreFile.LinkTarget is null && fileLength == 0) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + // Resolve symlinks + var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; + if (!resolvedFile.Exists) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var content = File.ReadAllText(resolvedFile.FullName); + if (string.IsNullOrWhiteSpace(content)) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var ignore = new Ignore.Ignore(); + var validRulesAdded = 0; + + foreach (var rule in rules) + { + try + { + ignore.Add(rule); + validRulesAdded++; + } + catch (RegexParseException) + { + // Ignore invalid patterns + } + } + + // No valid rules means treat as empty (ignore all) + return new ParsedIgnoreCacheEntry + { + Rules = ignore, + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = validRulesAdded == 0 + }; + } + + private static string GetPathToCheck(string path, bool isDirectory) + { + // Normalize Windows paths + var pathToCheck = IsWindows ? path.NormalizePath('/') : path; + + // Add trailing slash for directories to match "folder/" + if (isDirectory) + { + pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/"); + } + + return pathToCheck; + } + + private readonly record struct IgnoreFileCacheEntry(string? IgnoreFilePath); + + private sealed class ParsedIgnoreCacheEntry + { + public required Ignore.Ignore Rules { get; init; } + + public required DateTime FileLastModified { get; init; } + + public required long FileLength { get; init; } + + public required bool IsEmpty { get; init; } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 2bcb10e9e1..46facf881b 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -84,6 +84,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly FastConcurrentLru<Guid, BaseItem> _cache; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// <summary> /// The _root folder sync lock. @@ -125,6 +126,7 @@ namespace Emby.Server.Implementations.Library /// <param name="directoryService">The directory service.</param> /// <param name="peopleRepository">The people repository.</param> /// <param name="pathManager">The path manager.</param> + /// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param> public LibraryManager( IServerApplicationHost appHost, ILoggerFactory loggerFactory, @@ -146,7 +148,8 @@ namespace Emby.Server.Implementations.Library NamingOptions namingOptions, IDirectoryService directoryService, IPeopleRepository peopleRepository, - IPathManager pathManager) + IPathManager pathManager, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _appHost = appHost; _logger = loggerFactory.CreateLogger<LibraryManager>(); @@ -171,6 +174,7 @@ namespace Emby.Server.Implementations.Library _namingOptions = namingOptions; _peopleRepository = peopleRepository; _pathManager = pathManager; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -1303,6 +1307,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateMediaLibraryInternal(IProgress<double> progress, CancellationToken cancellationToken) { IsScanRunning = true; + ClearIgnoreRuleCache(); LibraryMonitor.Stop(); try @@ -1311,6 +1316,7 @@ namespace Emby.Server.Implementations.Library } finally { + ClearIgnoreRuleCache(); LibraryMonitor.Start(); IsScanRunning = false; } @@ -1318,6 +1324,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) { + ClearIgnoreRuleCache(); RootFolder.Children = null; await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); @@ -1360,6 +1367,14 @@ namespace Emby.Server.Implementations.Library { _persistenceService.DeleteItem(toDelete.ToArray()); } + + ClearIgnoreRuleCache(); + } + + /// <inheritdoc /> + public void ClearIgnoreRuleCache() + { + _dotIgnoreIgnoreRule.ClearDirectoryCache(); } private async Task PerformLibraryValidation(IProgress<double> progress, CancellationToken cancellationToken) @@ -3161,7 +3176,7 @@ namespace Emby.Server.Implementations.Library public IEnumerable<BaseItem> FindExtras(BaseItem owner, IReadOnlyList<FileSystemMetadata> fileSystemChildren, IDirectoryService directoryService) { // Apply .ignore rules - var filtered = fileSystemChildren.Where(c => !DotIgnoreIgnoreRule.IsIgnored(c, owner)).ToList(); + var filtered = fileSystemChildren.Where(c => !_dotIgnoreIgnoreRule.ShouldIgnore(c, owner)).ToList(); var isFolder = owner.IsFolder || (owner is Video video && (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd)); var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath); if (ownerVideoInfo is null) |