diff options
Diffstat (limited to 'Emby.Server.Implementations/Library')
| -rw-r--r-- | Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs | 273 | ||||
| -rw-r--r-- | Emby.Server.Implementations/Library/LibraryManager.cs | 19 |
2 files changed, 253 insertions, 39 deletions
diff --git a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs index ef5d24c70f..023c1e8915 100644 --- a/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs +++ b/Emby.Server.Implementations/Library/DotIgnoreIgnoreRule.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; +using BitFaster.Caching.Lru; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Resolvers; @@ -15,22 +17,36 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule { private static readonly bool IsWindows = OperatingSystem.IsWindows(); - private static FileInfo? FindIgnoreFile(DirectoryInfo directory) - { - for (var current = directory; current is not null; current = current.Parent) - { - var ignorePath = Path.Join(current.FullName, ".ignore"); - if (File.Exists(ignorePath)) - { - return new FileInfo(ignorePath); - } - } + private readonly FastConcurrentLru<string, IgnoreFileCacheEntry> _directoryCache; + private readonly FastConcurrentLru<string, ParsedIgnoreCacheEntry> _rulesCache; - return null; + /// <summary> + /// Initializes a new instance of the <see cref="DotIgnoreIgnoreRule"/> class. + /// </summary> + public DotIgnoreIgnoreRule() + { + var cacheSize = Math.Max(100, Environment.ProcessorCount * 100); + _directoryCache = new FastConcurrentLru<string, IgnoreFileCacheEntry>( + Environment.ProcessorCount, + cacheSize, + StringComparer.Ordinal); + _rulesCache = new FastConcurrentLru<string, ParsedIgnoreCacheEntry>( + Environment.ProcessorCount, + Math.Max(32, cacheSize / 4), + StringComparer.Ordinal); } /// <inheritdoc /> - public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnored(fileInfo, parent); + public bool ShouldIgnore(FileSystemMetadata fileInfo, BaseItem? parent) => IsIgnoredInternal(fileInfo, parent); + + /// <summary> + /// Clears the directory lookup cache. The parsed rules cache is not cleared + /// as it validates file modification time on each access. + /// </summary> + public void ClearDirectoryCache() + { + _directoryCache.Clear(); + } /// <summary> /// Checks whether or not the file is ignored. @@ -38,40 +54,38 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule /// <param name="fileInfo">The file information.</param> /// <param name="parent">The parent BaseItem.</param> /// <returns>True if the file should be ignored.</returns> - public static bool IsIgnored(FileSystemMetadata fileInfo, BaseItem? parent) + public bool IsIgnoredInternal(FileSystemMetadata fileInfo, BaseItem? parent) { var searchDirectory = fileInfo.IsDirectory - ? new DirectoryInfo(fileInfo.FullName) - : new DirectoryInfo(Path.GetDirectoryName(fileInfo.FullName) ?? string.Empty); + ? fileInfo.FullName + : Path.GetDirectoryName(fileInfo.FullName); - if (string.IsNullOrEmpty(searchDirectory.FullName)) + if (string.IsNullOrEmpty(searchDirectory)) { return false; } - var ignoreFile = FindIgnoreFile(searchDirectory); + var ignoreFile = FindIgnoreFileCached(searchDirectory); if (ignoreFile is null) { return false; } - // Fast path in case the ignore files isn't a symlink and is empty - if (ignoreFile.LinkTarget is null && ignoreFile.Length == 0) + var parsedEntry = GetParsedRules(ignoreFile); + if (parsedEntry is null) { - // Ignore directory if we just have the file - return true; + // File was deleted after we cached the path - clear the directory cache entry and return false + _directoryCache.TryRemove(searchDirectory, out _); + return false; } - var content = GetFileContent(ignoreFile); - return string.IsNullOrWhiteSpace(content) - || CheckIgnoreRules(fileInfo.FullName, content, fileInfo.IsDirectory); - } + // Empty file means ignore everything + if (parsedEntry.IsEmpty) + { + return true; + } - private static bool CheckIgnoreRules(string path, string ignoreFileContent, bool isDirectory) - { - // If file has content, base ignoring off the content .gitignore-style rules - var rules = ignoreFileContent.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - return CheckIgnoreRules(path, rules, isDirectory); + return parsedEntry.Rules.IsIgnored(GetPathToCheck(fileInfo.FullName, fileInfo.IsDirectory)); } /// <summary> @@ -117,8 +131,8 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return true; } - // Mitigate the problem of the Ignore library not handling Windows paths correctly. - // See https://github.com/jellyfin/jellyfin/issues/15484 + // Mitigate the problem of the Ignore library not handling Windows paths correctly. + // See https://github.com/jellyfin/jellyfin/issues/15484 var pathToCheck = normalizePath ? path.NormalizePath('/') : path; // Add trailing slash for directories to match "folder/" @@ -130,11 +144,196 @@ public class DotIgnoreIgnoreRule : IResolverIgnoreRule return ignore.IsIgnored(pathToCheck); } - private static string GetFileContent(FileInfo ignoreFile) + private FileInfo? FindIgnoreFileCached(string directory) + { + // Check if we have a cached result for this directory + if (_directoryCache.TryGet(directory, out var cached)) + { + return cached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(cached.IgnoreFileDirectory, ".ignore")); + } + + DirectoryInfo startDir; + try + { + startDir = new DirectoryInfo(directory); + } + catch (ArgumentException) + { + return null; + } + + // Walk up the directory tree to find .ignore file using DirectoryInfo.Parent + var checkedDirs = new List<string> { directory }; + + for (var current = startDir; current is not null; current = current.Parent) + { + var currentPath = current.FullName; + + // Check if this intermediate directory is cached + if (current != startDir && _directoryCache.TryGet(currentPath, out var parentCached)) + { + // Cache the result for all directories we checked + var entry = new IgnoreFileCacheEntry(parentCached.IgnoreFileDirectory); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return parentCached.IgnoreFileDirectory is null + ? null + : new FileInfo(Path.Join(parentCached.IgnoreFileDirectory, ".ignore")); + } + + var ignoreFile = new FileInfo(Path.Join(currentPath, ".ignore")); + if (ignoreFile.Exists) + { + // Cache for all directories we checked + var entry = new IgnoreFileCacheEntry(currentPath); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, entry); + } + + return ignoreFile; + } + + if (current != startDir) + { + checkedDirs.Add(currentPath); + } + } + + // No .ignore file found - cache null result for all directories + var nullEntry = new IgnoreFileCacheEntry((string?)null); + foreach (var dir in checkedDirs) + { + _directoryCache.AddOrUpdate(dir, nullEntry); + } + + return null; + } + + private ParsedIgnoreCacheEntry? GetParsedRules(FileInfo ignoreFile) + { + if (!ignoreFile.Exists) + { + _rulesCache.TryRemove(ignoreFile.FullName, out _); + return null; + } + + var lastModified = ignoreFile.LastWriteTimeUtc; + var fileLength = ignoreFile.Length; + var key = ignoreFile.FullName; + + // Check cache + if (_rulesCache.TryGet(key, out var cached)) + { + if (cached.FileLastModified == lastModified && cached.FileLength == fileLength) + { + return cached; + } + + // Stale - need to reparse + _rulesCache.TryRemove(key, out _); + } + + // Parse the file + var parsedEntry = ParseIgnoreFile(ignoreFile, lastModified, fileLength); + _rulesCache.AddOrUpdate(key, parsedEntry); + return parsedEntry; + } + + private static ParsedIgnoreCacheEntry ParseIgnoreFile(FileInfo ignoreFile, DateTime lastModified, long fileLength) { - ignoreFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; - return ignoreFile.Exists - ? File.ReadAllText(ignoreFile.FullName) - : string.Empty; + if (ignoreFile.LinkTarget is null && fileLength == 0) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + // Resolve symlinks + var resolvedFile = FileSystemHelper.ResolveLinkTarget(ignoreFile, returnFinalTarget: true) ?? ignoreFile; + if (!resolvedFile.Exists) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var content = File.ReadAllText(resolvedFile.FullName); + if (string.IsNullOrWhiteSpace(content)) + { + return new ParsedIgnoreCacheEntry + { + Rules = new Ignore.Ignore(), + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = true + }; + } + + var rules = content.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var ignore = new Ignore.Ignore(); + var validRulesAdded = 0; + + foreach (var rule in rules) + { + try + { + ignore.Add(rule); + validRulesAdded++; + } + catch (RegexParseException) + { + // Ignore invalid patterns + } + } + + // No valid rules means treat as empty (ignore all) + return new ParsedIgnoreCacheEntry + { + Rules = ignore, + FileLastModified = lastModified, + FileLength = fileLength, + IsEmpty = validRulesAdded == 0 + }; + } + + private static string GetPathToCheck(string path, bool isDirectory) + { + // Normalize Windows paths + var pathToCheck = IsWindows ? path.NormalizePath('/') : path; + + // Add trailing slash for directories to match "folder/" + if (isDirectory) + { + pathToCheck = string.Concat(pathToCheck.AsSpan().TrimEnd('/'), "/"); + } + + return pathToCheck; + } + + private readonly record struct IgnoreFileCacheEntry(string? IgnoreFileDirectory); + + private sealed class ParsedIgnoreCacheEntry + { + public required Ignore.Ignore Rules { get; init; } + + public required DateTime FileLastModified { get; init; } + + public required long FileLength { get; init; } + + public required bool IsEmpty { get; init; } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index b7852eeb4f..11f1496086 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -86,6 +86,7 @@ namespace Emby.Server.Implementations.Library private readonly ExtraResolver _extraResolver; private readonly IPathManager _pathManager; private readonly FastConcurrentLru<Guid, BaseItem> _cache; + private readonly DotIgnoreIgnoreRule _dotIgnoreIgnoreRule; /// <summary> /// The _root folder sync lock. @@ -127,6 +128,7 @@ namespace Emby.Server.Implementations.Library /// <param name="directoryService">The directory service.</param> /// <param name="peopleRepository">The people repository.</param> /// <param name="pathManager">The path manager.</param> + /// <param name="dotIgnoreIgnoreRule">The .ignore rule handler.</param> public LibraryManager( IServerApplicationHost appHost, ILoggerFactory loggerFactory, @@ -148,7 +150,8 @@ namespace Emby.Server.Implementations.Library NamingOptions namingOptions, IDirectoryService directoryService, IPeopleRepository peopleRepository, - IPathManager pathManager) + IPathManager pathManager, + DotIgnoreIgnoreRule dotIgnoreIgnoreRule) { _appHost = appHost; _logger = loggerFactory.CreateLogger<LibraryManager>(); @@ -173,6 +176,7 @@ namespace Emby.Server.Implementations.Library _namingOptions = namingOptions; _peopleRepository = peopleRepository; _pathManager = pathManager; + _dotIgnoreIgnoreRule = dotIgnoreIgnoreRule; _extraResolver = new ExtraResolver(loggerFactory.CreateLogger<ExtraResolver>(), namingOptions, directoryService); _configurationManager.ConfigurationUpdated += ConfigurationUpdated; @@ -1305,6 +1309,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateMediaLibraryInternal(IProgress<double> progress, CancellationToken cancellationToken) { IsScanRunning = true; + ClearIgnoreRuleCache(); LibraryMonitor.Stop(); try @@ -1313,6 +1318,7 @@ namespace Emby.Server.Implementations.Library } finally { + ClearIgnoreRuleCache(); LibraryMonitor.Start(); IsScanRunning = false; } @@ -1320,6 +1326,7 @@ namespace Emby.Server.Implementations.Library public async Task ValidateTopLibraryFolders(CancellationToken cancellationToken, bool removeRoot = false) { + ClearIgnoreRuleCache(); RootFolder.Children = null; await RootFolder.RefreshMetadata(cancellationToken).ConfigureAwait(false); @@ -1362,6 +1369,14 @@ namespace Emby.Server.Implementations.Library { _persistenceService.DeleteItem(toDelete.ToArray()); } + + ClearIgnoreRuleCache(); + } + + /// <inheritdoc /> + public void ClearIgnoreRuleCache() + { + _dotIgnoreIgnoreRule.ClearDirectoryCache(); } private async Task PerformLibraryValidation(IProgress<double> progress, CancellationToken cancellationToken) @@ -3182,7 +3197,7 @@ namespace Emby.Server.Implementations.Library public IEnumerable<BaseItem> FindExtras(BaseItem owner, IReadOnlyList<FileSystemMetadata> fileSystemChildren, IDirectoryService directoryService) { // Apply .ignore rules - var filtered = fileSystemChildren.Where(c => !DotIgnoreIgnoreRule.IsIgnored(c, owner)).ToList(); + var filtered = fileSystemChildren.Where(c => !_dotIgnoreIgnoreRule.ShouldIgnore(c, owner)).ToList(); var isFolder = owner.IsFolder || (owner is Video video && (video.VideoType == VideoType.BluRay || video.VideoType == VideoType.Dvd)); var ownerVideoInfo = VideoResolver.Resolve(owner.Path, isFolder, _namingOptions, libraryRoot: owner.ContainingFolderPath); if (ownerVideoInfo is null) |