diff options
Diffstat (limited to 'Emby.Server.Implementations/Data')
3 files changed, 589 insertions, 0 deletions
diff --git a/Emby.Server.Implementations/Data/BaseSqliteRepository.cs b/Emby.Server.Implementations/Data/BaseSqliteRepository.cs new file mode 100644 index 000000000..c7ac630a0 --- /dev/null +++ b/Emby.Server.Implementations/Data/BaseSqliteRepository.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Model.Logging; +using SQLitePCL.pretty; + +namespace Emby.Server.Implementations.Data +{ + public abstract class BaseSqliteRepository : IDisposable + { + protected string DbFilePath { get; set; } + protected SemaphoreSlim WriteLock = new SemaphoreSlim(1, 1); + protected ILogger Logger { get; private set; } + + protected BaseSqliteRepository(ILogger logger) + { + Logger = logger; + } + + protected virtual bool EnableConnectionPooling + { + get { return true; } + } + + protected virtual SQLiteDatabaseConnection CreateConnection(bool isReadOnly = false) + { + SQLite3.EnableSharedCache = false; + + ConnectionFlags connectionFlags; + + if (isReadOnly) + { + connectionFlags = ConnectionFlags.ReadOnly; + //connectionFlags = ConnectionFlags.Create; + //connectionFlags |= ConnectionFlags.ReadWrite; + } + else + { + connectionFlags = ConnectionFlags.Create; + connectionFlags |= ConnectionFlags.ReadWrite; + } + + if (EnableConnectionPooling) + { + connectionFlags |= ConnectionFlags.SharedCached; + } + else + { + connectionFlags |= ConnectionFlags.PrivateCache; + } + + connectionFlags |= ConnectionFlags.NoMutex; + + var db = SQLite3.Open(DbFilePath, connectionFlags, null); + + var queries = new[] + { + "PRAGMA page_size=4096", + "PRAGMA journal_mode=WAL", + "PRAGMA temp_store=memory", + "PRAGMA synchronous=Normal", + //"PRAGMA cache size=-10000" + }; + + //foreach (var query in queries) + //{ + // db.Execute(query); + //} + + db.ExecuteAll(string.Join(";", queries)); + + return db; + } + + private bool _disposed; + protected void CheckDisposed() + { + if (_disposed) + { + throw new ObjectDisposedException(GetType().Name + " has been disposed and cannot be accessed."); + } + } + + public void Dispose() + { + _disposed = true; + Dispose(true); + GC.SuppressFinalize(this); + } + + private readonly object _disposeLock = new object(); + + /// <summary> + /// Releases unmanaged and - optionally - managed resources. + /// </summary> + /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param> + protected virtual void Dispose(bool dispose) + { + if (dispose) + { + try + { + lock (_disposeLock) + { + WriteLock.Wait(); + + CloseConnection(); + } + } + catch (Exception ex) + { + Logger.ErrorException("Error disposing database", ex); + } + } + } + + protected virtual void CloseConnection() + { + + } + } +} diff --git a/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs b/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs new file mode 100644 index 000000000..dd32e2cbd --- /dev/null +++ b/Emby.Server.Implementations/Data/CleanDatabaseScheduledTask.cs @@ -0,0 +1,361 @@ +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Persistence; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.IO; +using MediaBrowser.Model.IO; +using MediaBrowser.Controller.Channels; +using MediaBrowser.Controller.Entities.Audio; +using MediaBrowser.Controller.IO; +using MediaBrowser.Controller.LiveTv; +using MediaBrowser.Controller.Net; +using MediaBrowser.Model.Globalization; +using MediaBrowser.Model.Tasks; +using Emby.Server.Implementations.ScheduledTasks; + +namespace Emby.Server.Implementations.Data +{ + public class CleanDatabaseScheduledTask : IScheduledTask + { + private readonly ILibraryManager _libraryManager; + private readonly IItemRepository _itemRepo; + private readonly ILogger _logger; + private readonly IServerConfigurationManager _config; + private readonly IFileSystem _fileSystem; + private readonly IHttpServer _httpServer; + private readonly ILocalizationManager _localization; + private readonly ITaskManager _taskManager; + + public const int MigrationVersion = 23; + public static bool EnableUnavailableMessage = false; + const int LatestSchemaVersion = 109; + + public CleanDatabaseScheduledTask(ILibraryManager libraryManager, IItemRepository itemRepo, ILogger logger, IServerConfigurationManager config, IFileSystem fileSystem, IHttpServer httpServer, ILocalizationManager localization, ITaskManager taskManager) + { + _libraryManager = libraryManager; + _itemRepo = itemRepo; + _logger = logger; + _config = config; + _fileSystem = fileSystem; + _httpServer = httpServer; + _localization = localization; + _taskManager = taskManager; + } + + public string Name + { + get { return "Clean Database"; } + } + + public string Description + { + get { return "Deletes obsolete content from the database."; } + } + + public string Category + { + get { return "Library"; } + } + + public async Task Execute(CancellationToken cancellationToken, IProgress<double> progress) + { + OnProgress(0); + + // Ensure these objects are lazy loaded. + // Without this there is a deadlock that will need to be investigated + var rootChildren = _libraryManager.RootFolder.Children.ToList(); + rootChildren = _libraryManager.GetUserRootFolder().Children.ToList(); + + var innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(p => + { + double newPercentCommplete = .4 * p; + OnProgress(newPercentCommplete); + + progress.Report(newPercentCommplete); + }); + + await UpdateToLatestSchema(cancellationToken, innerProgress).ConfigureAwait(false); + + innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(p => + { + double newPercentCommplete = 40 + .05 * p; + OnProgress(newPercentCommplete); + progress.Report(newPercentCommplete); + }); + await CleanDeadItems(cancellationToken, innerProgress).ConfigureAwait(false); + progress.Report(45); + + innerProgress = new ActionableProgress<double>(); + innerProgress.RegisterAction(p => + { + double newPercentCommplete = 45 + .55 * p; + OnProgress(newPercentCommplete); + progress.Report(newPercentCommplete); + }); + await CleanDeletedItems(cancellationToken, innerProgress).ConfigureAwait(false); + progress.Report(100); + + await _itemRepo.UpdateInheritedValues(cancellationToken).ConfigureAwait(false); + + if (_config.Configuration.MigrationVersion < MigrationVersion) + { + _config.Configuration.MigrationVersion = MigrationVersion; + _config.SaveConfiguration(); + } + + if (_config.Configuration.SchemaVersion < LatestSchemaVersion) + { + _config.Configuration.SchemaVersion = LatestSchemaVersion; + _config.SaveConfiguration(); + } + + if (EnableUnavailableMessage) + { + EnableUnavailableMessage = false; + _httpServer.GlobalResponse = null; + _taskManager.QueueScheduledTask<RefreshMediaLibraryTask>(); + } + + _taskManager.SuspendTriggers = false; + } + + private void OnProgress(double newPercentCommplete) + { + if (EnableUnavailableMessage) + { + var html = "<!doctype html><html><head><title>Emby</title></head><body>"; + var text = _localization.GetLocalizedString("DbUpgradeMessage"); + html += string.Format(text, newPercentCommplete.ToString("N2", CultureInfo.InvariantCulture)); + + html += "<script>setTimeout(function(){window.location.reload(true);}, 5000);</script>"; + html += "</body></html>"; + + _httpServer.GlobalResponse = html; + } + } + + private async Task UpdateToLatestSchema(CancellationToken cancellationToken, IProgress<double> progress) + { + var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + IsCurrentSchema = false, + ExcludeItemTypes = new[] { typeof(LiveTvProgram).Name } + }); + + var numComplete = 0; + var numItems = itemIds.Count; + + _logger.Debug("Upgrading schema for {0} items", numItems); + + var list = new List<BaseItem>(); + + foreach (var itemId in itemIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (itemId != Guid.Empty) + { + // Somehow some invalid data got into the db. It probably predates the boundary checking + var item = _libraryManager.GetItemById(itemId); + + if (item != null) + { + list.Add(item); + } + } + + if (list.Count >= 1000) + { + try + { + await _itemRepo.SaveItems(list, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.ErrorException("Error saving item", ex); + } + + list.Clear(); + } + + numComplete++; + double percent = numComplete; + percent /= numItems; + progress.Report(percent * 100); + } + + if (list.Count > 0) + { + try + { + await _itemRepo.SaveItems(list, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.ErrorException("Error saving item", ex); + } + } + + progress.Report(100); + } + + private async Task CleanDeadItems(CancellationToken cancellationToken, IProgress<double> progress) + { + var itemIds = _libraryManager.GetItemIds(new InternalItemsQuery + { + HasDeadParentId = true + }); + + var numComplete = 0; + var numItems = itemIds.Count; + + _logger.Debug("Cleaning {0} items with dead parent links", numItems); + + foreach (var itemId in itemIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + var item = _libraryManager.GetItemById(itemId); + + if (item != null) + { + _logger.Info("Cleaning item {0} type: {1} path: {2}", item.Name, item.GetType().Name, item.Path ?? string.Empty); + + await item.Delete(new DeleteOptions + { + DeleteFileLocation = false + + }).ConfigureAwait(false); + } + + numComplete++; + double percent = numComplete; + percent /= numItems; + progress.Report(percent * 100); + } + + progress.Report(100); + } + + private async Task CleanDeletedItems(CancellationToken cancellationToken, IProgress<double> progress) + { + var result = _itemRepo.GetItemIdsWithPath(new InternalItemsQuery + { + LocationTypes = new[] { LocationType.FileSystem }, + //Limit = limit, + + // These have their own cleanup routines + ExcludeItemTypes = new[] + { + typeof(Person).Name, + typeof(Genre).Name, + typeof(MusicGenre).Name, + typeof(GameGenre).Name, + typeof(Studio).Name, + typeof(Year).Name, + typeof(Channel).Name, + typeof(AggregateFolder).Name, + typeof(CollectionFolder).Name + } + }); + + var numComplete = 0; + var numItems = result.Items.Length; + + foreach (var item in result.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var path = item.Item2; + + try + { + if (_fileSystem.FileExists(path) || _fileSystem.DirectoryExists(path)) + { + continue; + } + + var libraryItem = _libraryManager.GetItemById(item.Item1); + + if (libraryItem.IsTopParent) + { + continue; + } + + var hasDualAccess = libraryItem as IHasDualAccess; + if (hasDualAccess != null && hasDualAccess.IsAccessedByName) + { + continue; + } + + var libraryItemPath = libraryItem.Path; + if (!string.Equals(libraryItemPath, path, StringComparison.OrdinalIgnoreCase)) + { + _logger.Error("CleanDeletedItems aborting delete for item {0}-{1} because paths don't match. {2}---{3}", libraryItem.Id, libraryItem.Name, libraryItem.Path ?? string.Empty, path ?? string.Empty); + continue; + } + + if (Folder.IsPathOffline(path)) + { + await libraryItem.UpdateIsOffline(true).ConfigureAwait(false); + continue; + } + + _logger.Info("Deleting item from database {0} because path no longer exists. type: {1} path: {2}", libraryItem.Name, libraryItem.GetType().Name, libraryItemPath ?? string.Empty); + + await libraryItem.OnFileDeleted().ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.ErrorException("Error in CleanDeletedItems. File {0}", ex, path); + } + + numComplete++; + double percent = numComplete; + percent /= numItems; + progress.Report(percent * 100); + } + } + + /// <summary> + /// Creates the triggers that define when the task will run + /// </summary> + /// <returns>IEnumerable{BaseTaskTrigger}.</returns> + public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() + { + return new[] { + + // Every so often + new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(24).Ticks} + }; + } + + public string Key + { + get { return "CleanDatabase"; } + } + } +}
\ No newline at end of file diff --git a/Emby.Server.Implementations/Data/SqliteExtensions.cs b/Emby.Server.Implementations/Data/SqliteExtensions.cs new file mode 100644 index 000000000..62615c669 --- /dev/null +++ b/Emby.Server.Implementations/Data/SqliteExtensions.cs @@ -0,0 +1,105 @@ +using System; +using System.Globalization; +using SQLitePCL.pretty; + +namespace Emby.Server.Implementations.Data +{ + public static class SqliteExtensions + { + public static void RunQueries(this SQLiteDatabaseConnection connection, string[] queries) + { + if (queries == null) + { + throw new ArgumentNullException("queries"); + } + + connection.RunInTransaction(conn => + { + //foreach (var query in queries) + //{ + // conn.Execute(query); + //} + conn.ExecuteAll(string.Join(";", queries)); + }); + } + + public static byte[] ToGuidParamValue(this string str) + { + return new Guid(str).ToByteArray(); + } + + public static Guid ReadGuid(this IResultSetValue result) + { + return new Guid(result.ToBlob()); + } + + public static string ToDateTimeParamValue(this DateTime dateValue) + { + var kind = DateTimeKind.Utc; + + return (dateValue.Kind == DateTimeKind.Unspecified) + ? DateTime.SpecifyKind(dateValue, kind).ToString( + GetDateTimeKindFormat(kind), + CultureInfo.InvariantCulture) + : dateValue.ToString( + GetDateTimeKindFormat(dateValue.Kind), + CultureInfo.InvariantCulture); + } + + private static string GetDateTimeKindFormat( + DateTimeKind kind) + { + return (kind == DateTimeKind.Utc) ? _datetimeFormatUtc : _datetimeFormatLocal; + } + + /// <summary> + /// An array of ISO-8601 DateTime formats that we support parsing. + /// </summary> + private static string[] _datetimeFormats = new string[] { + "THHmmssK", + "THHmmK", + "HH:mm:ss.FFFFFFFK", + "HH:mm:ssK", + "HH:mmK", + "yyyy-MM-dd HH:mm:ss.FFFFFFFK", /* NOTE: UTC default (5). */ + "yyyy-MM-dd HH:mm:ssK", + "yyyy-MM-dd HH:mmK", + "yyyy-MM-ddTHH:mm:ss.FFFFFFFK", + "yyyy-MM-ddTHH:mmK", + "yyyy-MM-ddTHH:mm:ssK", + "yyyyMMddHHmmssK", + "yyyyMMddHHmmK", + "yyyyMMddTHHmmssFFFFFFFK", + "THHmmss", + "THHmm", + "HH:mm:ss.FFFFFFF", + "HH:mm:ss", + "HH:mm", + "yyyy-MM-dd HH:mm:ss.FFFFFFF", /* NOTE: Non-UTC default (19). */ + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd HH:mm", + "yyyy-MM-ddTHH:mm:ss.FFFFFFF", + "yyyy-MM-ddTHH:mm", + "yyyy-MM-ddTHH:mm:ss", + "yyyyMMddHHmmss", + "yyyyMMddHHmm", + "yyyyMMddTHHmmssFFFFFFF", + "yyyy-MM-dd", + "yyyyMMdd", + "yy-MM-dd" + }; + + private static string _datetimeFormatUtc = _datetimeFormats[5]; + private static string _datetimeFormatLocal = _datetimeFormats[19]; + + public static DateTime ReadDateTime(this IResultSetValue result) + { + var dateText = result.ToString(); + + return DateTime.ParseExact( + dateText, _datetimeFormats, + DateTimeFormatInfo.InvariantInfo, + DateTimeStyles.None).ToUniversalTime(); + } + } +} |