From d20c775dafba32dce65f8d68fb6802732df00363 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Sun, 1 Feb 2026 23:07:01 +0100 Subject: Implement ignore rule caching --- .../Library/DotIgnoreIgnoreRule.cs | 272 ++++++++++++++++++--- .../Library/LibraryManager.cs | 19 +- 2 files changed, 251 insertions(+), 40 deletions(-) (limited to 'Emby.Server.Implementations/Library') diff --git a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs index ef5d24c70f..f073fc0bca 100644 --- a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs +++ b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Text.RegularExpressions; +using BitFaster.Caching.Lru; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Resolvers; @@ -15,22 +16,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule { private static readonly bool IsWindows = OperatingSystem.IsWindows(); - private static FileInfo? FindIgnoreFile(DirectoryInfo directory) - { - for (var current = directory; current is not null; current = current.Parent) - { - var ignorePath = Path.Join(current.FullName, ".ignore"); - if (File.Exists(ignorePath)) - { - return new FileInfo(ignorePath); - } - } + private readonly FastConcurrentLru _directoryCache; + private readonly FastConcurrentLru _rulesCache; - return null; + /// + /// Initializes a new instance of the class. + /// + public DotIgnoreIgnoreRule() + { + var cacheSize = Math.Max(100, Environment.ProcessorCount * 100); + _directoryCache = new FastConcurrentLru( + Environment.ProcessorCount, + cacheSize, + StringComparer.Ordinal); + _rulesCache = new FastConcurrentLru( + Environment.ProcessorCount, + Math.Max(32, cacheSize / 4), + StringComparer.Ordinal); } /// - public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent); + public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent); + + /// + /// Clears the directory lookup cache. The parsed rules cache is not cleared + /// as it validates file modification time on each access. + /// + public void ClearDirectoryCache() + { + _directoryCache.Clear(); + } /// /// Checks whether or not the file is ignored. @@ -38,40 +53,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule /// The file information. /// The parent BaseItem. /// True if the file should be ignored. - public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent) + public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent) { var searchDirectory = fileInfo.IsDirectory - ? new DirectoryInfo(fileInfo.FullName) - : new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty); + ? fileInfo.FullName + : Path.GetDirectoryName(fileInfo.FullName); - if (string.IsNullOrEmpty(searchDirectory.FullName)) + if (string.IsNullOrEmpty(searchDirectory)) { return false; } - var ignoreFile = FindIgnoreFile(searchDirectory); - if (ignoreFile is null) + var ignoreFilePath = FindIgnoreFileCached(searchDirectory); + if (ignoreFilePath is null) { return false; } - // Fast path in case the ignore files isn't a symlink and is empty - if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0) + var parsedEntry = GetParsedRules(ignoreFilePath); + if (parsedEntry is null) { - // Ignore directory if we just have the file - return true; + // File was deleted after we cached the path - clear the directory cache entry and return false + _directoryCache.TryRemove(searchDirectory, out _); + return false; } - var content = GetFileContent(ignoreFile); - return string.IsNullOrWhiteSpace(content) - || CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory); - } + // Empty file means ignore everything + if (parsedEntry.IsEmpty) + { + return true; + } - private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory) - { - // If file has content, base ignoring off the content .gitignore-style rules - var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - return CheckIgnoreRules(path, rules, isDirectory); + return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory)); } /// @@ -117,8 +130,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return true; } - // Mitigate the problem of the Ignore library not handling Windows paths correctly. - // See https://github.com/jellyfin/jellyfin/issues/15484 + // Mitigate the problem of the Ignore library not handling Windows paths correctly. + // See https://github.com/jellyfin/jellyfin/issues/15484 var pathToCheck = normalizePath ? path.NormalizePath('/') : path; // Add trailing slash for directories to match "folder/" @@ -130,11 +143,194 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return ignore.IsIgnored(pathToCheck); } - private static string GetFileContent(FileInfo ignoreFile) + private string? FindIgnoreFileCached(string directory) + { + // Check if we have a cached result for this directory + if (_directoryCache.TryGet(directory, out var cached)) + { + return cached.IgnoreFilePath; + } + + // Walk up the directory tree to find .ignore file + var current = directory; + var checkedDirs = new System.Collections.Generic.List { directory }; + + while (!string.IsNullOrEmpty(current)) + { + // Check if this intermediate directory is cached + if (current != directory && _directoryCache.TryGet(current, out var parentCached)) + { + // Cache the result for all directories we checked + var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFilePath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return parentCached.IgnoreFilePath; + } + + var ignorePath = Path.Join(current, ".ignore"); + if (File.Exists(ignorePath)) + { + // Cache for all directories we checked + var entry = new IgnoreFileCacheEntry(ignorePath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return ignorePath; + } + + var parent = Path.GetDirectoryName(current); + if (parent == current || string.IsNullOrEmpty(parent)) + { + break; + } + + current = parent; + checkedDirs.Add(current); + } + + // No .ignore file found - cache null result for all directories + var nullEntry = new IgnoreFileCacheEntry(null); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, nullEntry); + } + + return null; + } + + private ParsedIgnoreCacheEntry? GetParsedRules(string ignoreFilePath) { - ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; - return ignoreFile.Exists - ? File.ReadAllText(ignoreFile.FullName) - : string.Empty; + FileInfo fileInfo; + try + { + fileInfo = new FileInfo(ignoreFilePath); + if (!fileInfo.Exists) + { + _rulesCache.TryRemove(ignoreFilePath, out _); + return null; + } + } + catch + { + _rulesCache.TryRemove(ignoreFilePath, out _); + return null; + } + + var lastModified = fileInfo.LastWriteTimeUtc; + var fileLength = fileInfo.Length; + + // Check cache + if (_rulesCache.TryGet(ignoreFilePath, out var cached)) + { + if (cached.FileLastModified == lastModified && cached.FileLength == fileLength) + { + return cached; + } + + // Stale - need to reparse + _rulesCache.TryRemove(ignoreFilePath, out _); + } + + // Parse the file + var parsedEntry = ParseIgnoreFile(fileInfo, lastModified, fileLength); + _rulesCache.AddOrUpdate(ignoreFilePath, parsedEntry); + return parsedEntry; + } + + private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength) + { + if (ignoreFile.LinkTarget is null && fileLength == 0) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + // Resolve symlinks + var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; + if (!resolvedFile.Exists) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var content = File.ReadAllText(resolvedFile.FullName); + if (string.IsNullOrWhiteSpace(content)) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var ignore = new Ignore.Ignore(); + var validRulesAdded = 0; + + foreach (var rule in rules) + { + try + { + ignore.Add(rule); + validRulesAdded++; + } + catch (RegexParseException) + { + // Ignore invalid patterns + } + } + + // No valid rules means treat as empty (ignore all) + return new ParsedIgnoreCacheEntry + { + Rules = ignore, + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = validRulesAdded == 0 + }; + } + + private static string GetPathToCheck(string path, bool isDirectory) + { + // Normalize Windows paths + var pathToCheck = IsWindows ? path.NormalizePath('/') : path; + + // Add trailing slash for directories to match "folder/" + if (isDirectory) + { + pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/"); + } + + return pathToCheck; + } + + private readonly record struct IgnoreFileCacheEntry(string? IgnoreFilePath); + + private sealed class ParsedIgnoreCacheEntry + { + public required Ignore.Ignore Rules { get; init; } + + public required DateTime FileLastModified { get; init; } + + public required long FileLength { get; init; } + + public required bool IsEmpty { get; init; } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 2bcb10e9e1..46facf881b 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -84,6 +84,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly FastConcurrentLru _cache; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// /// The _root folder sync lock. @@ -125,6 +126,7 @@ namespace Emby.Server.Implementations.Library /// The directory service. /// The people repository. /// The path manager. + /// The .ignore rule handler. public LibraryManager( IServerApplicationHost appHost, ILoggerFactory loggerFactory, @@ -146,7 +148,8 @@ namespace Emby.Server.Implementations.Library NamingOptions namingOptions, IDirectoryService directoryService, IPeopleRepository peopleRepository, - IPathManager pathManager) + IPathManager pathManager, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _appHost = appHost; _logger = loggerFactory.CreateLogger(); @@ -171,6 +174,7 @@ namespace Emby.Server.Implementations.Library _namingOptions = namingOptions; _peopleRepository = peopleRepository; _pathManager = pathManager; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -1303,6 +1307,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateMediaLibraryInternal(IProgress progress, CancellationToken cancellationToken) { IsScanRunning = true; + ClearIgnoreRuleCache(); LibraryMonitor.Stop(); try @@ -1311,6 +1316,7 @@ namespace Emby.Server.Implementations.Library } finally { + ClearIgnoreRuleCache(); LibraryMonitor.Start(); IsScanRunning = false; } @@ -1318,6 +1324,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) { + ClearIgnoreRuleCache(); RootFolder.Children = null; await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); @@ -1360,6 +1367,14 @@ namespace Emby.Server.Implementations.Library { _persistenceService.DeleteItem(toDelete.ToArray()); } + + ClearIgnoreRuleCache(); + } + + /// + public void ClearIgnoreRuleCache() + { + _dotIgnoreIgnoreRule.ClearDirectoryCache(); } private async Task PerformLibraryValidation(IProgress progress, CancellationToken cancellationToken) @@ -3161,7 +3176,7 @@ namespace Emby.Server.Implementations.Library public IEnumerable FindExtras(BaseItem owner, IReadOnlyList fileSystemChildren, IDirectoryService directoryService) { // Apply .ignore rules - var filtered = fileSystemChildren.Where(c => !DotIgnoreIgnoreRule.IsIgnored(c, owner)).ToList(); + var filtered = fileSystemChildren.Where(c => !_dotIgnoreIgnoreRule.ShouldIgnore(c, owner)).ToList(); var isFolder = owner.IsFolder || (owner is Video video && (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd)); var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath); if (ownerVideoInfo is null) -- cgit v1.2.3 From 0f6bab03eb29030bbe0f00ed32ad72f97e321e62 Mon Sep 17 00:00:00 2001 From: Shadowghost Date: Mon, 4 May 2026 02:16:00 +0200 Subject: Fix Sonar comments --- .../Library/DotIgnoreIgnoreRule.cs | 89 +++++++++++----------- 1 file changed, 46 insertions(+), 43 deletions(-) (limited to 'Emby.Server.Implementations/Library') diff --git a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs index f073fc0bca..023c1e8915 100644 --- a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs +++ b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using BitFaster.Caching.Lru; @@ -64,13 +65,13 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return false; } - var ignoreFilePath = FindIgnoreFileCached(searchDirectory); - if (ignoreFilePath is null) + var ignoreFile = FindIgnoreFileCached(searchDirectory); + if (ignoreFile is null) { return false; } - var parsedEntry = GetParsedRules(ignoreFilePath); + var parsedEntry = GetParsedRules(ignoreFile); if (parsedEntry is null) { // File was deleted after we cached the path - clear the directory cache entry and return false @@ -143,58 +144,69 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return ignore.IsIgnored(pathToCheck); } - private string? FindIgnoreFileCached(string directory) + private FileInfo? FindIgnoreFileCached(string directory) { // Check if we have a cached result for this directory if (_directoryCache.TryGet(directory, out var cached)) { - return cached.IgnoreFilePath; + return cached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(cached.IgnoreFileDirectory, ".ignore")); } - // Walk up the directory tree to find .ignore file - var current = directory; - var checkedDirs = new System.Collections.Generic.List { directory }; + DirectoryInfo startDir; + try + { + startDir = new DirectoryInfo(directory); + } + catch (ArgumentException) + { + return null; + } + + // Walk up the directory tree to find .ignore file using DirectoryInfo.Parent + var checkedDirs = new List { directory }; - while (!string.IsNullOrEmpty(current)) + for (var current = startDir; current is not null; current = current.Parent) { + var currentPath = current.FullName; + // Check if this intermediate directory is cached - if (current != directory && _directoryCache.TryGet(current, out var parentCached)) + if (current != startDir && _directoryCache.TryGet(currentPath, out var parentCached)) { // Cache the result for all directories we checked - var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFilePath); + var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFileDirectory); foreach (var dir in checkedDirs) { _directoryCache.AddOrUpdate(dir, entry); } - return parentCached.IgnoreFilePath; + return parentCached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(parentCached.IgnoreFileDirectory, ".ignore")); } - var ignorePath = Path.Join(current, ".ignore"); - if (File.Exists(ignorePath)) + var ignoreFile = new FileInfo(Path.Join(currentPath, ".ignore")); + if (ignoreFile.Exists) { // Cache for all directories we checked - var entry = new IgnoreFileCacheEntry(ignorePath); + var entry = new IgnoreFileCacheEntry(currentPath); foreach (var dir in checkedDirs) { _directoryCache.AddOrUpdate(dir, entry); } - return ignorePath; + return ignoreFile; } - var parent = Path.GetDirectoryName(current); - if (parent == current || string.IsNullOrEmpty(parent)) + if (current != startDir) { - break; + checkedDirs.Add(currentPath); } - - current = parent; - checkedDirs.Add(current); } // No .ignore file found - cache null result for all directories - var nullEntry = new IgnoreFileCacheEntry(null); + var nullEntry = new IgnoreFileCacheEntry((string?)null); foreach (var dir in checkedDirs) { _directoryCache.AddOrUpdate(dir, nullEntry); @@ -203,29 +215,20 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return null; } - private ParsedIgnoreCacheEntry? GetParsedRules(string ignoreFilePath) + private ParsedIgnoreCacheEntry? GetParsedRules(FileInfo ignoreFile) { - FileInfo fileInfo; - try - { - fileInfo = new FileInfo(ignoreFilePath); - if (!fileInfo.Exists) - { - _rulesCache.TryRemove(ignoreFilePath, out _); - return null; - } - } - catch + if (!ignoreFile.Exists) { - _rulesCache.TryRemove(ignoreFilePath, out _); + _rulesCache.TryRemove(ignoreFile.FullName, out _); return null; } - var lastModified = fileInfo.LastWriteTimeUtc; - var fileLength = fileInfo.Length; + var lastModified = ignoreFile.LastWriteTimeUtc; + var fileLength = ignoreFile.Length; + var key = ignoreFile.FullName; // Check cache - if (_rulesCache.TryGet(ignoreFilePath, out var cached)) + if (_rulesCache.TryGet(key, out var cached)) { if (cached.FileLastModified == lastModified && cached.FileLength == fileLength) { @@ -233,12 +236,12 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule } // Stale - need to reparse - _rulesCache.TryRemove(ignoreFilePath, out _); + _rulesCache.TryRemove(key, out _); } // Parse the file - var parsedEntry = ParseIgnoreFile(fileInfo, lastModified, fileLength); - _rulesCache.AddOrUpdate(ignoreFilePath, parsedEntry); + var parsedEntry = ParseIgnoreFile(ignoreFile, lastModified, fileLength); + _rulesCache.AddOrUpdate(key, parsedEntry); return parsedEntry; } @@ -321,7 +324,7 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return pathToCheck; } - private readonly record struct IgnoreFileCacheEntry(string? IgnoreFilePath); + private readonly record struct IgnoreFileCacheEntry(string? IgnoreFileDirectory); private sealed class ParsedIgnoreCacheEntry { -- cgit v1.2.3