diff options
Diffstat (limited to 'MediaBrowser.Server.Implementations')
7 files changed, 88 insertions, 426 deletions
diff --git a/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs index f069d40c1..a4d99ae17 100644 --- a/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs +++ b/MediaBrowser.Server.Implementations/IO/DirectoryWatchers.cs @@ -66,7 +66,7 @@ namespace MediaBrowser.Server.Implementations.IO public async void RemoveTempIgnore(string path) { // This is an arbitraty amount of time, but delay it because file system writes often trigger events after RemoveTempIgnore has been called. - await Task.Delay(1000).ConfigureAwait(false); + await Task.Delay(1500).ConfigureAwait(false); string val; _tempIgnoredPaths.TryRemove(path, out val); diff --git a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs index 1bc3f1094..084f9b05c 100644 --- a/MediaBrowser.Server.Implementations/Library/LibraryManager.cs +++ b/MediaBrowser.Server.Implementations/Library/LibraryManager.cs @@ -709,65 +709,6 @@ namespace MediaBrowser.Server.Implementations.Library } /// <summary> - /// Generically retrieves an IBN item - /// </summary> - /// <typeparam name="T"></typeparam> - /// <param name="path">The path.</param> - /// <param name="name">The name.</param> - /// <param name="cancellationToken">The cancellation token.</param> - /// <param name="allowSlowProviders">if set to <c>true</c> [allow slow providers].</param> - /// <param name="refreshMetadata">if set to <c>true</c> [force creation].</param> - /// <returns>Task{``0}.</returns> - /// <exception cref="System.ArgumentNullException"> - /// </exception> - private async Task<T> GetItemByName<T>(string path, string name, CancellationToken cancellationToken, bool allowSlowProviders = true, bool refreshMetadata = false) - where T : BaseItem, new() - { - if (string.IsNullOrEmpty(path)) - { - throw new ArgumentNullException(); - } - - if (string.IsNullOrEmpty(name)) - { - throw new ArgumentNullException(); - } - - var validFilename = FileSystem.GetValidFilename(name); - - var key = Path.Combine(path, validFilename); - - BaseItem obj; - - if (!_itemsByName.TryGetValue(key, out obj)) - { - var tuple = CreateItemByName<T>(key, name); - - obj = tuple.Item2; - - _itemsByName.AddOrUpdate(key, obj, (keyName, oldValue) => obj); - - try - { - await obj.RefreshMetadata(cancellationToken, tuple.Item1, allowSlowProviders: allowSlowProviders).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - BaseItem removed; - _itemsByName.TryRemove(key, out removed); - - throw; - } - } - else if (refreshMetadata) - { - await obj.RefreshMetadata(cancellationToken, false, allowSlowProviders: allowSlowProviders).ConfigureAwait(false); - } - - return obj as T; - } - - /// <summary> /// Creates an IBN item based on a given path /// </summary> /// <typeparam name="T"></typeparam> @@ -846,7 +787,7 @@ namespace MediaBrowser.Server.Implementations.Library await item.RefreshMetadata(cancellationToken).ConfigureAwait(false); } - catch (IOException ex) + catch (Exception ex) { _logger.ErrorException("Error validating IBN entry {0}", ex, person.Name); } @@ -963,12 +904,16 @@ namespace MediaBrowser.Server.Implementations.Library progress.Report(2); + var innerProgress = new ActionableProgress<double>(); + + innerProgress.RegisterAction(pct => progress.Report(2 + pct * .13)); + // Run prescan tasks - await RunPrescanTasks(progress, cancellationToken).ConfigureAwait(false); + await RunPrescanTasks(innerProgress, cancellationToken).ConfigureAwait(false); progress.Report(15); - var innerProgress = new ActionableProgress<double>(); + innerProgress = new ActionableProgress<double>(); innerProgress.RegisterAction(pct => progress.Report(15 + pct * .6)); @@ -977,8 +922,12 @@ namespace MediaBrowser.Server.Implementations.Library progress.Report(75); + innerProgress = new ActionableProgress<double>(); + + innerProgress.RegisterAction(pct => progress.Report(75 + pct * .25)); + // Run post-scan tasks - await RunPostScanTasks(progress, cancellationToken).ConfigureAwait(false); + await RunPostScanTasks(innerProgress, cancellationToken).ConfigureAwait(false); progress.Report(100); @@ -995,41 +944,45 @@ namespace MediaBrowser.Server.Implementations.Library /// <returns>Task.</returns> private async Task RunPrescanTasks(IProgress<double> progress, CancellationToken cancellationToken) { - var prescanTasks = PrescanTasks.ToList(); - var progressDictionary = new Dictionary<ILibraryPrescanTask, double>(); + var tasks = PrescanTasks.ToList(); + + var numComplete = 0; + var numTasks = tasks.Count; - var tasks = prescanTasks.Select(i => Task.Run(async () => + foreach (var task in tasks) { var innerProgress = new ActionableProgress<double>(); + // Prevent access to modified closure + var currentNumComplete = numComplete; + innerProgress.RegisterAction(pct => { - lock (progressDictionary) - { - progressDictionary[i] = pct; - - double percent = progressDictionary.Values.Sum(); - percent /= prescanTasks.Count; - - progress.Report(2 + percent * .13); - } + double innerPercent = (currentNumComplete * 100) + pct; + innerPercent /= numTasks; + progress.Report(innerPercent); }); try { - await i.Run(innerProgress, cancellationToken); + await task.Run(innerProgress, cancellationToken); } catch (OperationCanceledException) { - _logger.Info("Pre-scan task cancelled: {0}", i.GetType().Name); + _logger.Info("Pre-scan task cancelled: {0}", task.GetType().Name); } catch (Exception ex) { - _logger.ErrorException("Error running prescan task", ex); + _logger.ErrorException("Error running pre-scan task", ex); } - })); - await Task.WhenAll(tasks).ConfigureAwait(false); + numComplete++; + double percent = numComplete; + percent /= numTasks; + progress.Report(percent * 100); + } + + progress.Report(100); } /// <summary> @@ -1040,41 +993,45 @@ namespace MediaBrowser.Server.Implementations.Library /// <returns>Task.</returns> private async Task RunPostScanTasks(IProgress<double> progress, CancellationToken cancellationToken) { - var postscanTasks = PostscanTasks.ToList(); - var progressDictionary = new Dictionary<ILibraryPostScanTask, double>(); + var tasks = PostscanTasks.ToList(); - var tasks = postscanTasks.Select(i => Task.Run(async () => + var numComplete = 0; + var numTasks = tasks.Count; + + foreach (var task in tasks) { var innerProgress = new ActionableProgress<double>(); + // Prevent access to modified closure + var currentNumComplete = numComplete; + innerProgress.RegisterAction(pct => { - lock (progressDictionary) - { - progressDictionary[i] = pct; - - double percent = progressDictionary.Values.Sum(); - percent /= postscanTasks.Count; - - progress.Report(75 + percent * .25); - } + double innerPercent = (currentNumComplete * 100) + pct; + innerPercent /= numTasks; + progress.Report(innerPercent); }); try { - await i.Run(innerProgress, cancellationToken); + await task.Run(innerProgress, cancellationToken); } catch (OperationCanceledException) { - _logger.Info("Post-scan task cancelled: {0}", i.GetType().Name); + _logger.Info("Post-scan task cancelled: {0}", task.GetType().Name); } catch (Exception ex) { _logger.ErrorException("Error running postscan task", ex); } - })); - await Task.WhenAll(tasks).ConfigureAwait(false); + numComplete++; + double percent = numComplete; + percent /= numTasks; + progress.Report(percent * 100); + } + + progress.Report(100); } /// <summary> @@ -1288,15 +1245,15 @@ namespace MediaBrowser.Server.Implementations.Library /// <returns>Task.</returns> public async Task UpdateItem(BaseItem item, ItemUpdateType updateReason, CancellationToken cancellationToken) { - await ItemRepository.SaveItem(item, cancellationToken).ConfigureAwait(false); - - UpdateItemInLibraryCache(item); - if (item.LocationType == LocationType.FileSystem) { await SaveMetadata(item, updateReason).ConfigureAwait(false); } + await ItemRepository.SaveItem(item, cancellationToken).ConfigureAwait(false); + + UpdateItemInLibraryCache(item); + if (ItemUpdated != null) { try @@ -1346,7 +1303,16 @@ namespace MediaBrowser.Server.Implementations.Library /// <returns>BaseItem.</returns> public BaseItem RetrieveItem(Guid id) { - return ItemRepository.RetrieveItem(id); + var item = ItemRepository.RetrieveItem(id); + + var folder = item as Folder; + + if (folder != null) + { + folder.LoadSavedChildren(); + } + + return item; } private readonly ConcurrentDictionary<string, SemaphoreSlim> _fileLocks = new ConcurrentDictionary<string, SemaphoreSlim>(); diff --git a/MediaBrowser.Server.Implementations/Library/LuceneSearchEngine.cs b/MediaBrowser.Server.Implementations/Library/LuceneSearchEngine.cs index ca878476f..5ef04ef9d 100644 --- a/MediaBrowser.Server.Implementations/Library/LuceneSearchEngine.cs +++ b/MediaBrowser.Server.Implementations/Library/LuceneSearchEngine.cs @@ -1,10 +1,4 @@ -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Documents; -using Lucene.Net.Index; -using Lucene.Net.QueryParsers; -using Lucene.Net.Search; -using Lucene.Net.Store; -using MediaBrowser.Controller; +using MediaBrowser.Controller; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; using MediaBrowser.Controller.Library; @@ -30,37 +24,6 @@ namespace MediaBrowser.Server.Implementations.Library _libraryManager = libraryManager; _logger = logManager.GetLogger("Lucene"); - - //string luceneDbPath = serverPaths.DataPath + "\\SearchIndexDB"; - //if (!System.IO.Directory.Exists(luceneDbPath)) - // System.IO.Directory.CreateDirectory(luceneDbPath); - //else if(File.Exists(luceneDbPath + "\\write.lock")) - // File.Delete(luceneDbPath + "\\write.lock"); - - //LuceneSearch.Init(luceneDbPath, _logger); - - //BaseItem.LibraryManager.LibraryChanged += LibraryChanged; - } - - //public void LibraryChanged(object source, ChildrenChangedEventArgs changeInformation) - //{ - // Task.Run(() => - // { - // if (changeInformation.ItemsAdded.Count + changeInformation.ItemsUpdated.Count > 0) - // { - // LuceneSearch.AddUpdateLuceneIndex(changeInformation.ItemsAdded.Concat(changeInformation.ItemsUpdated)); - // } - - // if (changeInformation.ItemsRemoved.Count > 0) - // { - // LuceneSearch.RemoveFromLuceneIndex(changeInformation.ItemsRemoved); - // } - // }); - //} - - public void AddItemsToIndex(IEnumerable<BaseItem> items) - { - LuceneSearch.AddUpdateLuceneIndex(items); } /// <summary> @@ -72,15 +35,7 @@ namespace MediaBrowser.Server.Implementations.Library /// <exception cref="System.ArgumentNullException">searchTerm</exception> public IEnumerable<BaseItem> Search(IEnumerable<BaseItem> items, string searchTerm) { - if (string.IsNullOrEmpty(searchTerm)) - { - throw new ArgumentNullException("searchTerm"); - } - - var hits = LuceneSearch.Search(searchTerm, items.Count()); - - //return hits; - return hits.Where(searchHit => items.Any(p => p.Id == searchHit.Id)); + return items; } public void Dispose() @@ -362,231 +317,4 @@ namespace MediaBrowser.Server.Implementations.Library return term.Split().Where(i => !string.IsNullOrWhiteSpace(i)).ToList(); } } - - public static class LuceneSearch - { - private static ILogger logger; - - private static string path; - private static object lockOb = new object(); - - private static FSDirectory _directory; - private static FSDirectory directory - { - get - { - if (_directory == null) - { - logger.Info("Opening new Directory: " + path); - _directory = FSDirectory.Open(path); - } - return _directory; - } - set - { - _directory = value; - } - } - - private static IndexWriter _writer; - private static IndexWriter writer - { - get - { - if (_writer == null) - { - logger.Info("Opening new IndexWriter"); - _writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED); - } - return _writer; - } - set - { - _writer = value; - } - } - - private static Dictionary<string, float> bonusTerms; - - public static void Init(string path, ILogger logger) - { - logger.Info("Lucene: Init"); - - bonusTerms = new Dictionary<string, float>(); - bonusTerms.Add("Name", 2); - bonusTerms.Add("Overview", 1); - - // Optimize the DB on initialization - // TODO: Test whether this has.. - // Any effect what-so-ever (apart from initializing the indexwriter on the mainthread context, which makes things a whole lot easier) - // Costs too much time - // Is heavy on the CPU / Memory - - LuceneSearch.logger = logger; - LuceneSearch.path = path; - - writer.Optimize(); - } - - private static StandardAnalyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30); - - private static Searcher searcher = null; - - private static Document createDocument(BaseItem data) - { - Document doc = new Document(); - - doc.Add(new Field("Id", data.Id.ToString(), Field.Store.YES, Field.Index.NO)); - doc.Add(new Field("Name", data.Name, Field.Store.YES, Field.Index.ANALYZED) { Boost = 2 }); - doc.Add(new Field("Overview", data.Overview != null ? data.Overview : "", Field.Store.YES, Field.Index.ANALYZED)); - - return doc; - } - - private static void Create(BaseItem item) - { - lock (lockOb) - { - try - { - if (searcher != null) - { - try - { - searcher.Dispose(); - } - catch (Exception e) - { - logger.ErrorException("Error in Lucene while creating index (disposing alive searcher)", e, item); - } - - searcher = null; - } - - _removeFromLuceneIndex(item); - _addToLuceneIndex(item); - } - catch (Exception e) - { - logger.ErrorException("Error in Lucene while creating index", e, item); - } - } - } - - private static void _addToLuceneIndex(BaseItem data) - { - // Prevent double entries - var doc = createDocument(data); - - writer.AddDocument(doc); - } - - private static void _removeFromLuceneIndex(BaseItem data) - { - var query = new TermQuery(new Term("Id", data.Id.ToString())); - writer.DeleteDocuments(query); - } - - public static void AddUpdateLuceneIndex(IEnumerable<BaseItem> items) - { - foreach (var item in items) - { - logger.Info("Adding/Updating BaseItem " + item.Name + "(" + item.Id.ToString() + ") to/on Lucene Index"); - Create(item); - } - - writer.Commit(); - writer.Flush(true, true, true); - } - - public static void RemoveFromLuceneIndex(IEnumerable<BaseItem> items) - { - foreach (var item in items) - { - logger.Info("Removing BaseItem " + item.Name + "(" + item.Id.ToString() + ") from Lucene Index"); - _removeFromLuceneIndex(item); - } - - writer.Commit(); - writer.Flush(true, true, true); - } - - public static IEnumerable<BaseItem> Search(string searchQuery, int maxHits) - { - var results = new List<BaseItem>(); - - lock (lockOb) - { - try - { - if (searcher == null) - { - searcher = new IndexSearcher(directory, true); - } - - BooleanQuery finalQuery = new BooleanQuery(); - - MultiFieldQueryParser parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { "Name", "Overview" }, analyzer, bonusTerms); - - string[] terms = searchQuery.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); - foreach (string term in terms) - finalQuery.Add(parser.Parse(term.Replace("~", "") + "~0.75"), Occur.SHOULD); - foreach (string term in terms) - finalQuery.Add(parser.Parse(term.Replace("*", "") + "*"), Occur.SHOULD); - - logger.Debug("Querying Lucene with query: " + finalQuery.ToString()); - - long start = DateTime.Now.Ticks; - var searchResult = searcher.Search(finalQuery, maxHits); - foreach (var searchHit in searchResult.ScoreDocs) - { - Document hit = searcher.Doc(searchHit.Doc); - results.Add(BaseItem.LibraryManager.GetItemById(Guid.Parse(hit.Get("Id")))); - } - long total = DateTime.Now.Ticks - start; - float msTotal = (float)total / TimeSpan.TicksPerMillisecond; - logger.Debug(searchResult.ScoreDocs.Length + " result" + (searchResult.ScoreDocs.Length == 1 ? "" : "s") + " in " + msTotal + " ms."); - } - catch (Exception e) - { - logger.ErrorException("Error while searching Lucene index", e); - } - } - - return results; - } - - public static void CloseAll() - { - logger.Debug("Lucene: CloseAll"); - if (writer != null) - { - logger.Debug("Lucene: CloseAll - Writer is alive"); - writer.Flush(true, true, true); - writer.Commit(); - writer.WaitForMerges(); - writer.Dispose(); - writer = null; - } - if (analyzer != null) - { - logger.Debug("Lucene: CloseAll - Analyzer is alive"); - analyzer.Close(); - analyzer.Dispose(); - analyzer = null; - } - if (searcher != null) - { - logger.Debug("Lucene: CloseAll - Searcher is alive"); - searcher.Dispose(); - searcher = null; - } - if (directory != null) - { - logger.Debug("Lucene: CloseAll - Directory is alive"); - directory.Dispose(); - directory = null; - } - } - } } diff --git a/MediaBrowser.Server.Implementations/Library/ResolverHelper.cs b/MediaBrowser.Server.Implementations/Library/ResolverHelper.cs index 8b98a6944..a6b13f8dd 100644 --- a/MediaBrowser.Server.Implementations/Library/ResolverHelper.cs +++ b/MediaBrowser.Server.Implementations/Library/ResolverHelper.cs @@ -20,7 +20,7 @@ namespace MediaBrowser.Server.Implementations.Library /// <param name="args">The args.</param> public static void SetInitialItemValues(BaseItem item, ItemResolveArgs args) { - item.ResolveArgs = args; + item.ResetResolveArgs(args); // If the resolver didn't specify this if (string.IsNullOrEmpty(item.Path)) diff --git a/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs b/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs index 73909de27..7c964eacb 100644 --- a/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs +++ b/MediaBrowser.Server.Implementations/Library/Validators/ArtistsValidator.cs @@ -185,7 +185,7 @@ namespace MediaBrowser.Server.Implementations.Library.Validators /// <param name="cancellationToken">The cancellation token.</param> /// <param name="progress">The progress.</param> /// <returns>Task{Artist[]}.</returns> - private async Task<ConcurrentBag<Artist>> GetAllArtists(IEnumerable<Audio> allSongs, CancellationToken cancellationToken, IProgress<double> progress) + private async Task<List<Artist>> GetAllArtists(IEnumerable<Audio> allSongs, CancellationToken cancellationToken, IProgress<double> progress) { var allArtists = allSongs .SelectMany(i => @@ -203,60 +203,36 @@ namespace MediaBrowser.Server.Implementations.Library.Validators .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); - const int maxTasks = 3; - - var tasks = new List<Task>(); - - var returnArtists = new ConcurrentBag<Artist>(); + var returnArtists = new List<Artist>(allArtists.Count); var numComplete = 0; var numArtists = allArtists.Count; foreach (var artist in allArtists) { - if (tasks.Count > maxTasks) - { - await Task.WhenAll(tasks).ConfigureAwait(false); - tasks.Clear(); - - // Safe cancellation point, when there are no pending tasks - cancellationToken.ThrowIfCancellationRequested(); - } - - // Avoid accessing the foreach variable within the closure - var currentArtist = artist; + cancellationToken.ThrowIfCancellationRequested(); - tasks.Add(Task.Run(async () => + try { - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var artistItem = _libraryManager.GetArtist(currentArtist); + var artistItem = _libraryManager.GetArtist(artist); - await artistItem.RefreshMetadata(cancellationToken).ConfigureAwait(false); + await artistItem.RefreshMetadata(cancellationToken).ConfigureAwait(false); - returnArtists.Add(artistItem); - } - catch (IOException ex) - { - _logger.ErrorException("Error validating Artist {0}", ex, currentArtist); - } + returnArtists.Add(artistItem); + } + catch (IOException ex) + { + _logger.ErrorException("Error validating Artist {0}", ex, artist); + } - // Update progress - lock (progress) - { - numComplete++; - double percent = numComplete; - percent /= numArtists; + // Update progress + numComplete++; + double percent = numComplete; + percent /= numArtists; - progress.Report(100 * percent); - } - })); + progress.Report(100 * percent); } - await Task.WhenAll(tasks).ConfigureAwait(false); - return returnArtists; } diff --git a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj index f409b7205..86e250fe4 100644 --- a/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj +++ b/MediaBrowser.Server.Implementations/MediaBrowser.Server.Implementations.csproj @@ -37,12 +37,6 @@ <Reference Include="Alchemy"> <HintPath>..\packages\Alchemy.2.2.1\lib\net40\Alchemy.dll</HintPath> </Reference> - <Reference Include="ICSharpCode.SharpZipLib"> - <HintPath>..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll</HintPath> - </Reference> - <Reference Include="Lucene.Net"> - <HintPath>..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll</HintPath> - </Reference> <Reference Include="System" /> <Reference Include="System.Core" /> <Reference Include="System.Drawing" /> diff --git a/MediaBrowser.Server.Implementations/packages.config b/MediaBrowser.Server.Implementations/packages.config index 5d9463408..d4d92f09f 100644 --- a/MediaBrowser.Server.Implementations/packages.config +++ b/MediaBrowser.Server.Implementations/packages.config @@ -1,7 +1,6 @@ <?xml version="1.0" encoding="utf-8"?> <packages> <package id="Alchemy" version="2.2.1" targetFramework="net45" /> - <package id="Lucene.Net" version="3.0.3" targetFramework="net45" /> <package id="MediaBrowser.BdInfo" version="1.0.0.2" targetFramework="net45" /> <package id="morelinq" version="1.0.16006" targetFramework="net45" /> <package id="Rx-Core" version="2.1.30214.0" targetFramework="net45" /> @@ -13,6 +12,5 @@ <package id="ServiceStack.OrmLite.SqlServer" version="3.9.43" targetFramework="net45" /> <package id="ServiceStack.Redis" version="3.9.43" targetFramework="net45" /> <package id="ServiceStack.Text" version="3.9.62" targetFramework="net45" /> - <package id="SharpZipLib" version="0.86.0" targetFramework="net45" /> <package id="System.Data.SQLite.x86" version="1.0.88.0" targetFramework="net45" /> </packages>
\ No newline at end of file |