Upgrade Swashbuckle and fix OpenAPI spec (#15886)

5 files changed, 71 insertions, 46 deletions

diff --git a/Directory.Packages.props b/Directory.Packages.props
index 14ff3fadb..f71027e1a 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -80,8 +80,8 @@
     <PackageVersion Include="SmartAnalyzers.MultithreadingAnalyzer" Version="1.1.31" />
     <PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.556" />
     <PackageVersion Include="Svg.Skia" Version="3.2.1" />
-    <PackageVersion Include="Swashbuckle.AspNetCore.ReDoc" Version="6.5.0" />
-    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.2.3" />
+    <PackageVersion Include="Swashbuckle.AspNetCore.ReDoc" Version="6.9.0" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.9.0" />
     <PackageVersion Include="System.Globalization" Version="4.3.0" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.3" />
     <PackageVersion Include="System.Text.Encoding.CodePages" Version="9.0.11" />
diff --git a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs
index 04dd19eda..8373fd50f 100644
--- a/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs
+++ b/Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs
@@ -255,6 +255,7 @@ namespace Jellyfin.Server.Extensions
                 c.AddSwaggerTypeMappings();
 
                 c.SchemaFilter<IgnoreEnumSchemaFilter>();
+                c.SchemaFilter<FlagsEnumSchemaFilter>();
                 c.OperationFilter<RetryOnTemporarilyUnavailableFilter>();
                 c.OperationFilter<SecurityRequirementsOperationFilter>();
                 c.OperationFilter<FileResponseFilter>();
@@ -342,25 +343,6 @@ namespace Jellyfin.Server.Extensions
                     }
                 });
 
-            /*
-             * Support BlurHash dictionary
-             */
-            options.MapType<Dictionary<ImageType, Dictionary<string, string>>>(() =>
-                new OpenApiSchema
-                {
-                    Type = "object",
-                    Properties = typeof(ImageType).GetEnumNames().ToDictionary(
-                        name => name,
-                        _ => new OpenApiSchema
-                        {
-                            Type = "object",
-                            AdditionalProperties = new OpenApiSchema
-                            {
-                                Type = "string"
-                            }
-                        })
-                });
-
             // Support dictionary with nullable string value.
             options.MapType<Dictionary<string, string?>>(() =>
                 new OpenApiSchema
@@ -373,21 +355,6 @@ namespace Jellyfin.Server.Extensions
                     }
                 });
 
-            // Manually describe Flags enum.
-            options.MapType<TranscodeReason>(() =>
-                new OpenApiSchema
-                {
-                    Type = "array",
-                    Items = new OpenApiSchema
-                    {
-                        Reference = new OpenApiReference
-                        {
-                            Id = nameof(TranscodeReason),
-                            Type = ReferenceType.Schema,
-                        }
-                    }
-                });
-
             // Swashbuckle doesn't use JsonOptions to describe responses, so we need to manually describe it.
             options.MapType<Version>(() => new OpenApiSchema
             {
diff --git a/Jellyfin.Server/Filters/AdditionalModelFilter.cs b/Jellyfin.Server/Filters/AdditionalModelFilter.cs
index 58d37db5a..7407bd2eb 100644
--- a/Jellyfin.Server/Filters/AdditionalModelFilter.cs
+++ b/Jellyfin.Server/Filters/AdditionalModelFilter.cs
@@ -225,15 +225,6 @@ namespace Jellyfin.Server.Filters
 
                 context.SchemaGenerator.GenerateSchema(configuration.ConfigurationType, context.SchemaRepository);
             }
-
-            context.SchemaRepository.AddDefinition(nameof(TranscodeReason), new OpenApiSchema
-            {
-                Type = "string",
-                Enum = Enum.GetNames<TranscodeReason>()
-                    .Select(e => new OpenApiString(e))
-                    .Cast<IOpenApiAny>()
-                    .ToArray()
-            });
         }
     }
 }
diff --git a/Jellyfin.Server/Filters/CachingOpenApiProvider.cs b/Jellyfin.Server/Filters/CachingOpenApiProvider.cs
index b560ec50e..833b68444 100644
--- a/Jellyfin.Server/Filters/CachingOpenApiProvider.cs
+++ b/Jellyfin.Server/Filters/CachingOpenApiProvider.cs
@@ -2,6 +2,7 @@ using System;
 using AsyncKeyedLock;
 using Microsoft.AspNetCore.Mvc.ApiExplorer;
 using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.OpenApi.Models;
 using Swashbuckle.AspNetCore.Swagger;
@@ -23,6 +24,7 @@ internal sealed class CachingOpenApiProvider : ISwaggerProvider
     private readonly IMemoryCache _memoryCache;
     private readonly SwaggerGenerator _swaggerGenerator;
     private readonly SwaggerGeneratorOptions _swaggerGeneratorOptions;
+    private readonly ILogger<CachingOpenApiProvider> _logger;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="CachingOpenApiProvider"/> class.
@@ -31,15 +33,18 @@ internal sealed class CachingOpenApiProvider : ISwaggerProvider
     /// <param name="apiDescriptionsProvider">The api descriptions provider.</param>
     /// <param name="schemaGenerator">The schema generator.</param>
     /// <param name="memoryCache">The memory cache.</param>
+    /// <param name="logger">The logger.</param>
     public CachingOpenApiProvider(
         IOptions<SwaggerGeneratorOptions> optionsAccessor,
         IApiDescriptionGroupCollectionProvider apiDescriptionsProvider,
         ISchemaGenerator schemaGenerator,
-        IMemoryCache memoryCache)
+        IMemoryCache memoryCache,
+        ILogger<CachingOpenApiProvider> logger)
     {
         _swaggerGeneratorOptions = optionsAccessor.Value;
         _swaggerGenerator = new SwaggerGenerator(_swaggerGeneratorOptions, apiDescriptionsProvider, schemaGenerator);
         _memoryCache = memoryCache;
+        _logger = logger;
     }
 
     /// <inheritdoc />
@@ -61,7 +66,16 @@ internal sealed class CachingOpenApiProvider : ISwaggerProvider
             throw new InvalidOperationException("OpenApi document is generating");
         }
 
+        try
+        {
         openApiDocument = _swaggerGenerator.GetSwagger(documentName);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "OpenAPI generation error");
+            throw;
+        }
+
         _memoryCache.Set(CacheKey, openApiDocument, _cacheOptions);
         return AdjustDocument(openApiDocument, host, basePath);
     }
diff --git a/Jellyfin.Server/Filters/FlagsEnumSchemaFilter.cs b/Jellyfin.Server/Filters/FlagsEnumSchemaFilter.cs
new file mode 100644
index 000000000..3e0b69d01
--- /dev/null
+++ b/Jellyfin.Server/Filters/FlagsEnumSchemaFilter.cs
@@ -0,0 +1,53 @@
+using System;
+using Microsoft.OpenApi.Models;
+using Swashbuckle.AspNetCore.SwaggerGen;
+
+namespace Jellyfin.Server.Filters;
+
+/// <summary>
+/// Schema filter to ensure flags enums are represented correctly in OpenAPI.
+/// </summary>
+/// <remarks>
+/// For flags enums:
+/// - The enum schema definition is set to type "string" (not integer).
+/// - Properties using flags enums are transformed to arrays referencing the enum schema.
+/// </remarks>
+public class FlagsEnumSchemaFilter : ISchemaFilter
+{
+    /// <inheritdoc />
+    public void Apply(OpenApiSchema schema, SchemaFilterContext context)
+    {
+        var type = context.Type.IsEnum ? context.Type : Nullable.GetUnderlyingType(context.Type);
+        if (type is null || !type.IsEnum)
+        {
+            return;
+        }
+
+        // Check if enum has [Flags] attribute
+        if (!type.IsDefined(typeof(FlagsAttribute), false))
+        {
+            return;
+        }
+
+        if (context.MemberInfo is null)
+        {
+            // Processing the enum definition itself - ensure it's type "string" not "integer"
+            schema.Type = "string";
+            schema.Format = null;
+        }
+        else
+        {
+            // Processing a property that uses the flags enum - transform to array
+            // Generate the enum schema to ensure it exists in the repository
+            var enumSchema = context.SchemaGenerator.GenerateSchema(type, context.SchemaRepository);
+
+            // Flags enums should be represented as arrays referencing the enum schema
+            // since multiple values can be combined
+            schema.Type = "array";
+            schema.Format = null;
+            schema.Enum = null;
+            schema.AllOf = null;
+            schema.Items = enumSchema;
+        }
+    }
+}