Merge pull request #2535 from mark-monteiro/logging-migration

Create Logging Configuration Heirarchy

8 files changed, 139 insertions, 82 deletions

diff --git a/Jellyfin.Server/Migrations/IMigrationRoutine.cs b/Jellyfin.Server/Migrations/IMigrationRoutine.cs
new file mode 100644
index 000000000..eab995d67
--- /dev/null
+++ b/Jellyfin.Server/Migrations/IMigrationRoutine.cs
@@ -0,0 +1,28 @@
+using System;
+using Microsoft.Extensions.Logging;
+
+namespace Jellyfin.Server.Migrations
+{
+    /// <summary>
+    /// Interface that describes a migration routine.
+    /// </summary>
+    internal interface IMigrationRoutine
+    {
+        /// <summary>
+        /// Gets the unique id for this migration. This should never be modified after the migration has been created.
+        /// </summary>
+        public Guid Id { get; }
+
+        /// <summary>
+        /// Gets the display name of the migration.
+        /// </summary>
+        public string Name { get; }
+
+        /// <summary>
+        /// Execute the migration routine.
+        /// </summary>
+        /// <param name="host">Host that hosts current version.</param>
+        /// <param name="logger">Host logger.</param>
+        public void Perform(CoreAppHost host, ILogger logger);
+    }
+}
diff --git a/Jellyfin.Server/Migrations/IUpdater.cs b/Jellyfin.Server/Migrations/IUpdater.cs
deleted file mode 100644
index 9b749841c..000000000
--- a/Jellyfin.Server/Migrations/IUpdater.cs
+++ /dev/null
@@ -1,23 +0,0 @@
-using System;
-using Microsoft.Extensions.Logging;
-
-namespace Jellyfin.Server.Migrations
-{
-    /// <summary>
-    /// Interface that descibes a migration routine.
-    /// </summary>
-    internal interface IUpdater
-    {
-        /// <summary>
-        /// Gets the name of the migration, must be unique.
-        /// </summary>
-        public abstract string Name { get; }
-
-        /// <summary>
-        /// Execute the migration routine.
-        /// </summary>
-        /// <param name="host">Host that hosts current version.</param>
-        /// <param name="logger">Host logger.</param>
-        public abstract void Perform(CoreAppHost host, ILogger logger);
-    }
-}
diff --git a/Jellyfin.Server/Migrations/MigrationOptions.cs b/Jellyfin.Server/Migrations/MigrationOptions.cs
index 6b7831158..816dd9ee7 100644
--- a/Jellyfin.Server/Migrations/MigrationOptions.cs
+++ b/Jellyfin.Server/Migrations/MigrationOptions.cs
@@ -1,3 +1,6 @@
+using System;
+using System.Collections.Generic;
+
 namespace Jellyfin.Server.Migrations
 {
     /// <summary>
@@ -10,14 +13,12 @@ namespace Jellyfin.Server.Migrations
         /// </summary>
         public MigrationOptions()
         {
-            Applied = System.Array.Empty<string>();
+            Applied = new List<(Guid Id, string Name)>();
         }
 
-#pragma warning disable CA1819 // Properties should not return arrays
         /// <summary>
-        /// Gets or sets the list of applied migration routine names.
+        /// Gets the list of applied migration routine names.
         /// </summary>
-        public string[] Applied { get; set; }
-#pragma warning restore CA1819 // Properties should not return arrays
+        public List<(Guid Id, string Name)> Applied { get; }
     }
 }
diff --git a/Jellyfin.Server/Migrations/MigrationRunner.cs b/Jellyfin.Server/Migrations/MigrationRunner.cs
index ca4c79cfd..b5ea04dca 100644
--- a/Jellyfin.Server/Migrations/MigrationRunner.cs
+++ b/Jellyfin.Server/Migrations/MigrationRunner.cs
@@ -13,9 +13,10 @@ namespace Jellyfin.Server.Migrations
         /// <summary>
         /// The list of known migrations, in order of applicability.
         /// </summary>
-        internal static readonly IUpdater[] Migrations =
+        internal static readonly IMigrationRoutine[] Migrations =
         {
-            new Routines.DisableTranscodingThrottling()
+            new Routines.DisableTranscodingThrottling(),
+            new Routines.CreateUserLoggingConfigFile()
         };
 
         /// <summary>
@@ -28,47 +29,44 @@ namespace Jellyfin.Server.Migrations
             var logger = loggerFactory.CreateLogger<MigrationRunner>();
             var migrationOptions = ((IConfigurationManager)host.ServerConfigurationManager).GetConfiguration<MigrationOptions>(MigrationsListStore.StoreKey);
 
-            if (!host.ServerConfigurationManager.Configuration.IsStartupWizardCompleted && migrationOptions.Applied.Length == 0)
+            if (!host.ServerConfigurationManager.Configuration.IsStartupWizardCompleted && migrationOptions.Applied.Count == 0)
             {
                 // If startup wizard is not finished, this is a fresh install.
                 // Don't run any migrations, just mark all of them as applied.
-                logger.LogInformation("Marking all known migrations as applied because this is fresh install");
-                migrationOptions.Applied = Migrations.Select(m => m.Name).ToArray();
+                logger.LogInformation("Marking all known migrations as applied because this is a fresh install");
+                migrationOptions.Applied.AddRange(Migrations.Select(m => (m.Id, m.Name)));
                 host.ServerConfigurationManager.SaveConfiguration(MigrationsListStore.StoreKey, migrationOptions);
                 return;
             }
 
-            var applied = migrationOptions.Applied.ToList();
+            var appliedMigrationIds = migrationOptions.Applied.Select(m => m.Id).ToHashSet();
 
             for (var i = 0; i < Migrations.Length; i++)
             {
-                var updater = Migrations[i];
-                if (applied.Contains(updater.Name))
+                var migrationRoutine = Migrations[i];
+                if (appliedMigrationIds.Contains(migrationRoutine.Id))
                 {
-                    logger.LogDebug("Skipping migration '{Name}' since it is already applied", updater.Name);
+                    logger.LogDebug("Skipping migration '{Name}' since it is already applied", migrationRoutine.Name);
                     continue;
                 }
 
-                logger.LogInformation("Applying migration '{Name}'", updater.Name);
+                logger.LogInformation("Applying migration '{Name}'", migrationRoutine.Name);
+
                 try
                 {
-                    updater.Perform(host, logger);
+                    migrationRoutine.Perform(host, logger);
                 }
                 catch (Exception ex)
                 {
-                    logger.LogError(ex, "Could not apply migration '{Name}'", updater.Name);
+                    logger.LogError(ex, "Could not apply migration '{Name}'", migrationRoutine.Name);
                     throw;
                 }
 
-                logger.LogInformation("Migration '{Name}' applied successfully", updater.Name);
-                applied.Add(updater.Name);
-            }
-
-            if (applied.Count > migrationOptions.Applied.Length)
-            {
-                logger.LogInformation("Some migrations were run, saving the state");
-                migrationOptions.Applied = applied.ToArray();
+                // Mark the migration as completed
+                logger.LogInformation("Migration '{Name}' applied successfully", migrationRoutine.Name);
+                migrationOptions.Applied.Add((migrationRoutine.Id, migrationRoutine.Name));
                 host.ServerConfigurationManager.SaveConfiguration(MigrationsListStore.StoreKey, migrationOptions);
+                logger.LogDebug("Migration '{Name}' marked as applied in configuration.", migrationRoutine.Name);
             }
         }
     }
diff --git a/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs
new file mode 100644
index 000000000..3bc32c047
--- /dev/null
+++ b/Jellyfin.Server/Migrations/Routines/CreateUserLoggingConfigFile.cs
@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using MediaBrowser.Common.Configuration;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
+
+namespace Jellyfin.Server.Migrations.Routines
+{
+    /// <summary>
+    /// Migration to initialize the user logging configuration file "logging.user.json".
+    /// If the deprecated logging.json file exists and has a custom config, it will be used as logging.user.json,
+    /// otherwise a blank file will be created.
+    /// </summary>
+    internal class CreateUserLoggingConfigFile : IMigrationRoutine
+    {
+        /// <summary>
+        /// File history for logging.json as existed during this migration creation. The contents for each has been minified.
+        /// </summary>
+        private readonly List<string> _defaultConfigHistory = new List<string>
+        {
+            // 9a6c27947353585391e211aa88b925f81e8cd7b9
+            @"{""Serilog"":{""MinimumLevel"":{""Default"":""Information"",""Override"":{""Microsoft"":""Warning"",""System"":""Warning""}},""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message:lj}{NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""retainedFileCountLimit"":3,""rollOnFileSizeLimit"":true,""fileSizeLimitBytes"":100000000,""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message}{NewLine}{Exception}""}}]}}],""Enrich"":[""FromLogContext"",""WithThreadId""]}}",
+            // 71bdcd730705a714ee208eaad7290b7c68df3885
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message:lj}{NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""retainedFileCountLimit"":3,""rollOnFileSizeLimit"":true,""fileSizeLimitBytes"":100000000,""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message}{NewLine}{Exception}""}}]}}],""Enrich"":[""FromLogContext"",""WithThreadId""]}}",
+            // a44936f97f8afc2817d3491615a7cfe1e31c251c
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] {Message:lj}{NewLine}{Exception}""}},{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] {Message}{NewLine}{Exception}""}}]}}",
+            // 7af3754a11ad5a4284f107997fb5419a010ce6f3
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] {Message:lj}{NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] {Message}{NewLine}{Exception}""}}]}}]}}",
+            // 60691349a11f541958e0b2247c9abc13cb40c9fb
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] {Message:lj}{NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""retainedFileCountLimit"":3,""rollOnFileSizeLimit"":true,""fileSizeLimitBytes"":100000000,""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] {Message}{NewLine}{Exception}""}}]}}]}}",
+            // 65fe243afbcc4b596cf8726708c1965cd34b5f68
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] {ThreadId} {SourceContext}: {Message:lj} {NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""retainedFileCountLimit"":3,""rollOnFileSizeLimit"":true,""fileSizeLimitBytes"":100000000,""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] {ThreadId} {SourceContext}:{Message} {NewLine}{Exception}""}}]}}],""Enrich"":[""FromLogContext"",""WithThreadId""]}}",
+            // 96c9af590494aa8137d5a061aaf1e68feee60b67
+            @"{""Serilog"":{""MinimumLevel"":""Information"",""WriteTo"":[{""Name"":""Console"",""Args"":{""outputTemplate"":""[{Timestamp:HH:mm:ss}] [{Level:u3}] [{ThreadId}] {SourceContext}: {Message:lj}{NewLine}{Exception}""}},{""Name"":""Async"",""Args"":{""configure"":[{""Name"":""File"",""Args"":{""path"":""%JELLYFIN_LOG_DIR%//log_.log"",""rollingInterval"":""Day"",""retainedFileCountLimit"":3,""rollOnFileSizeLimit"":true,""fileSizeLimitBytes"":100000000,""outputTemplate"":""[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz}] [{Level:u3}] [{ThreadId}] {SourceContext}:{Message}{NewLine}{Exception}""}}]}}],""Enrich"":[""FromLogContext"",""WithThreadId""]}}",
+        };
+
+        /// <inheritdoc/>
+        public Guid Id => Guid.Parse("{EF103419-8451-40D8-9F34-D1A8E93A1679}");
+
+        /// <inheritdoc/>
+        public string Name => "CreateLoggingConfigHeirarchy";
+
+        /// <inheritdoc/>
+        public void Perform(CoreAppHost host, ILogger logger)
+        {
+            var logDirectory = host.Resolve<IApplicationPaths>().ConfigurationDirectoryPath;
+            var existingConfigPath = Path.Combine(logDirectory, "logging.json");
+
+            // If the existing logging.json config file is unmodified, then 'reset' it by moving it to 'logging.old.json'
+            // NOTE: This config file has 'reloadOnChange: true', so this change will take effect immediately even though it has already been loaded
+            if (File.Exists(existingConfigPath) && ExistingConfigUnmodified(existingConfigPath))
+            {
+                File.Move(existingConfigPath, Path.Combine(logDirectory, "logging.old.json"));
+            }
+        }
+
+        /// <summary>
+        /// Check if the existing logging.json file has not been modified by the user by comparing it to all the
+        /// versions in our git history. Until now, the file has never been migrated after first creation so users
+        /// could have any version from the git history.
+        /// </summary>
+        /// <exception cref="IOException"><paramref name="oldConfigPath"/> does not exist or could not be read.</exception>
+        private bool ExistingConfigUnmodified(string oldConfigPath)
+        {
+            var existingConfigJson = JToken.Parse(File.ReadAllText(oldConfigPath));
+            return _defaultConfigHistory
+                .Select(historicalConfigText => JToken.Parse(historicalConfigText))
+                .Any(historicalConfigJson => JToken.DeepEquals(existingConfigJson, historicalConfigJson));
+        }
+    }
+}
diff --git a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs
index 936c3640e..673f0e415 100644
--- a/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs
+++ b/Jellyfin.Server/Migrations/Routines/DisableTranscodingThrottling.cs
@@ -10,9 +10,12 @@ namespace Jellyfin.Server.Migrations.Routines
     /// <summary>
     /// Disable transcode throttling for all installations since it is currently broken for certain video formats.
     /// </summary>
-    internal class DisableTranscodingThrottling : IUpdater
+    internal class DisableTranscodingThrottling : IMigrationRoutine
     {
         /// <inheritdoc/>
+        public Guid Id => Guid.Parse("{4124C2CD-E939-4FFB-9BE9-9B311C413638}");
+
+        /// <inheritdoc/>
         public string Name => "DisableTranscodingThrottling";
 
         /// <inheritdoc/>
diff --git a/Jellyfin.Server/Migrations/Routines/DisableZealousLogging.cs b/Jellyfin.Server/Migrations/Routines/DisableZealousLogging.cs
deleted file mode 100644
index 501f8f865..000000000
--- a/Jellyfin.Server/Migrations/Routines/DisableZealousLogging.cs
+++ /dev/null
@@ -1,29 +0,0 @@
-using System;
-using System.IO;
-using MediaBrowser.Common.Configuration;
-using MediaBrowser.Model.Configuration;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
-using Serilog;
-using ILogger = Microsoft.Extensions.Logging.ILogger;
-
-namespace Jellyfin.Server.Migrations.Routines
-{
-    /// <summary>
-    /// Updater that takes care of bringing configuration up to 10.5.0 standards.
-    /// </summary>
-    internal class DisableZealousLogging : IUpdater
-    {
-        /// <inheritdoc/>
-        public string Name => "DisableZealousLogging";
-
-        /// <inheritdoc/>
-        // This tones down logging from some components
-        public void Perform(CoreAppHost host, ILogger logger)
-        {
-            string configPath = Path.Combine(host.ServerConfigurationManager.ApplicationPaths.ConfigurationDirectoryPath, Program.LoggingConfigFile);
-            // TODO: fix up the config
-            throw new NotImplementedException("don't know how to fix logging yet");
-        }
-    }
-}
diff --git a/Jellyfin.Server/Program.cs b/Jellyfin.Server/Program.cs
index 027186105..7c3d0f277 100644
--- a/Jellyfin.Server/Program.cs
+++ b/Jellyfin.Server/Program.cs
@@ -39,9 +39,14 @@ namespace Jellyfin.Server
     public static class Program
     {
         /// <summary>
-        /// The name of logging configuration file.
+        /// The name of logging configuration file containing application defaults.
         /// </summary>
-        public static readonly string LoggingConfigFile = "logging.json";
+        public static readonly string LoggingConfigFileDefault = "logging.default.json";
+
+        /// <summary>
+        /// The name of the logging configuration file containing the system-specific override settings.
+        /// </summary>
+        public static readonly string LoggingConfigFileSystem = "logging.json";
 
         private static readonly CancellationTokenSource _tokenSource = new CancellationTokenSource();
         private static readonly ILoggerFactory _loggerFactory = new SerilogLoggerFactory();
@@ -443,7 +448,7 @@ namespace Jellyfin.Server
         private static async Task<IConfiguration> CreateConfiguration(IApplicationPaths appPaths)
         {
             const string ResourcePath = "Jellyfin.Server.Resources.Configuration.logging.json";
-            string configPath = Path.Combine(appPaths.ConfigurationDirectoryPath, LoggingConfigFile);
+            string configPath = Path.Combine(appPaths.ConfigurationDirectoryPath, LoggingConfigFileDefault);
 
             if (!File.Exists(configPath))
             {
@@ -465,7 +470,8 @@ namespace Jellyfin.Server
             return new ConfigurationBuilder()
                 .SetBasePath(appPaths.ConfigurationDirectoryPath)
                 .AddInMemoryCollection(ConfigurationOptions.Configuration)
-                .AddJsonFile(LoggingConfigFile, false, true)
+                .AddJsonFile(LoggingConfigFileDefault, optional: false, reloadOnChange: true)
+                .AddJsonFile(LoggingConfigFileSystem, optional: true, reloadOnChange: true)
                 .AddEnvironmentVariables("JELLYFIN_")
                 .Build();
         }