diff options
Diffstat (limited to 'src')
62 files changed, 4492 insertions, 0 deletions
diff --git a/src/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj b/src/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj new file mode 100644 index 000000000..034691322 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj @@ -0,0 +1,45 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <!-- ProjectGuid is only included as a requirement for SonarQube analysis --> + <PropertyGroup> + <ProjectGuid>{154872D9-6C12-4007-96E3-8F70A58386CE}</ProjectGuid> + </PropertyGroup> + + <PropertyGroup> + <TargetFramework>net7.0</TargetFramework> + <GenerateAssemblyInfo>false</GenerateAssemblyInfo> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <ItemGroup> + <Compile Include="..\..\SharedVersion.cs" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="BlurHashSharp" /> + <PackageReference Include="BlurHashSharp.SkiaSharp" /> + <PackageReference Include="SkiaSharp" /> + <PackageReference Include="SkiaSharp.NativeAssets.Linux" /> + <PackageReference Include="SkiaSharp.Svg" /> + <PackageReference Include="SkiaSharp.HarfBuzz" /> + <PackageReference Include="HarfBuzzSharp.NativeAssets.Linux" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="..\..\MediaBrowser.Model\MediaBrowser.Model.csproj" /> + <ProjectReference Include="..\..\MediaBrowser.Controller\MediaBrowser.Controller.csproj" /> + <ProjectReference Include="..\..\MediaBrowser.Common\MediaBrowser.Common.csproj" /> + </ItemGroup> + + <!-- Code analysers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + <PackageReference Include="SerilogAnalyzer" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" PrivateAssets="All" /> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.Drawing.Skia/PercentPlayedDrawer.cs b/src/Jellyfin.Drawing.Skia/PercentPlayedDrawer.cs new file mode 100644 index 000000000..e2e90be47 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/PercentPlayedDrawer.cs @@ -0,0 +1,35 @@ +using System; +using MediaBrowser.Model.Drawing; +using SkiaSharp; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Static helper class used to draw percentage-played indicators on images. +/// </summary> +public static class PercentPlayedDrawer +{ + private const int IndicatorHeight = 8; + + /// <summary> + /// Draw a percentage played indicator on a canvas. + /// </summary> + /// <param name="canvas">The canvas to draw the indicator on.</param> + /// <param name="imageSize">The size of the image being drawn on.</param> + /// <param name="percent">The percentage played to display with the indicator.</param> + public static void Process(SKCanvas canvas, ImageDimensions imageSize, double percent) + { + using var paint = new SKPaint(); + var endX = imageSize.Width - 1; + var endY = imageSize.Height - 1; + + paint.Color = SKColor.Parse("#99000000"); + paint.Style = SKPaintStyle.Fill; + canvas.DrawRect(SKRect.Create(0, (float)endY - IndicatorHeight, endX, endY), paint); + + double foregroundWidth = (endX * percent) / 100; + + paint.Color = SKColor.Parse("#FF00A4DC"); + canvas.DrawRect(SKRect.Create(0, (float)endY - IndicatorHeight, Convert.ToInt32(foregroundWidth), endY), paint); + } +} diff --git a/src/Jellyfin.Drawing.Skia/Properties/AssemblyInfo.cs b/src/Jellyfin.Drawing.Skia/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..e7db09449 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/Properties/AssemblyInfo.cs @@ -0,0 +1,21 @@ +using System.Reflection; +using System.Resources; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Jellyfin.Drawing.Skia")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Jellyfin Project")] +[assembly: AssemblyProduct("Jellyfin Server")] +[assembly: AssemblyCopyright("Copyright © 2019 Jellyfin Contributors. Code released under the GNU General Public License")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: NeutralResourcesLanguage("en")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] diff --git a/src/Jellyfin.Drawing.Skia/SkiaCodecException.cs b/src/Jellyfin.Drawing.Skia/SkiaCodecException.cs new file mode 100644 index 000000000..581fa000d --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/SkiaCodecException.cs @@ -0,0 +1,44 @@ +using System.Globalization; +using SkiaSharp; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Represents errors that occur during interaction with Skia codecs. +/// </summary> +public class SkiaCodecException : SkiaException +{ + /// <summary> + /// Initializes a new instance of the <see cref="SkiaCodecException" /> class. + /// </summary> + /// <param name="result">The non-successful codec result returned by Skia.</param> + public SkiaCodecException(SKCodecResult result) + { + CodecResult = result; + } + + /// <summary> + /// Initializes a new instance of the <see cref="SkiaCodecException" /> class + /// with a specified error message. + /// </summary> + /// <param name="result">The non-successful codec result returned by Skia.</param> + /// <param name="message">The message that describes the error.</param> + public SkiaCodecException(SKCodecResult result, string message) + : base(message) + { + CodecResult = result; + } + + /// <summary> + /// Gets the non-successful codec result returned by Skia. + /// </summary> + public SKCodecResult CodecResult { get; } + + /// <inheritdoc /> + public override string ToString() + => string.Format( + CultureInfo.InvariantCulture, + "Non-success codec result: {0}\n{1}", + CodecResult, + base.ToString()); +} diff --git a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs new file mode 100644 index 000000000..5a1d3dc5f --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -0,0 +1,548 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using BlurHashSharp.SkiaSharp; +using Jellyfin.Extensions; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Drawing; +using MediaBrowser.Model.Drawing; +using Microsoft.Extensions.Logging; +using SkiaSharp; +using SKSvg = SkiaSharp.Extended.Svg.SKSvg; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Image encoder that uses <see cref="SkiaSharp"/> to manipulate images. +/// </summary> +public class SkiaEncoder : IImageEncoder +{ + private static readonly HashSet<string> _transparentImageTypes = new(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" }; + + private readonly ILogger<SkiaEncoder> _logger; + private readonly IApplicationPaths _appPaths; + + /// <summary> + /// Initializes a new instance of the <see cref="SkiaEncoder"/> class. + /// </summary> + /// <param name="logger">The application logger.</param> + /// <param name="appPaths">The application paths.</param> + public SkiaEncoder(ILogger<SkiaEncoder> logger, IApplicationPaths appPaths) + { + _logger = logger; + _appPaths = appPaths; + } + + /// <inheritdoc/> + public string Name => "Skia"; + + /// <inheritdoc/> + public bool SupportsImageCollageCreation => true; + + /// <inheritdoc/> + public bool SupportsImageEncoding => true; + + /// <inheritdoc/> + public IReadOnlyCollection<string> SupportedInputFormats => + new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + "jpeg", + "jpg", + "png", + "dng", + "webp", + "gif", + "bmp", + "ico", + "astc", + "ktx", + "pkm", + "wbmp", + // TODO: check if these are supported on multiple platforms + // https://github.com/google/skia/blob/master/infra/bots/recipes/test.py#L454 + // working on windows at least + "cr2", + "nef", + "arw" + }; + + /// <inheritdoc/> + public IReadOnlyCollection<ImageFormat> SupportedOutputFormats + => new HashSet<ImageFormat> { ImageFormat.Webp, ImageFormat.Jpg, ImageFormat.Png }; + + /// <summary> + /// Check if the native lib is available. + /// </summary> + /// <returns>True if the native lib is available, otherwise false.</returns> + public static bool IsNativeLibAvailable() + { + try + { + // test an operation that requires the native library + SKPMColor.PreMultiply(SKColors.Black); + return true; + } + catch (Exception) + { + return false; + } + } + + /// <summary> + /// Convert a <see cref="ImageFormat"/> to a <see cref="SKEncodedImageFormat"/>. + /// </summary> + /// <param name="selectedFormat">The format to convert.</param> + /// <returns>The converted format.</returns> + public static SKEncodedImageFormat GetImageFormat(ImageFormat selectedFormat) + { + return selectedFormat switch + { + ImageFormat.Bmp => SKEncodedImageFormat.Bmp, + ImageFormat.Jpg => SKEncodedImageFormat.Jpeg, + ImageFormat.Gif => SKEncodedImageFormat.Gif, + ImageFormat.Webp => SKEncodedImageFormat.Webp, + _ => SKEncodedImageFormat.Png + }; + } + + /// <inheritdoc /> + /// <exception cref="FileNotFoundException">The path is not valid.</exception> + public ImageDimensions GetImageSize(string path) + { + if (!File.Exists(path)) + { + throw new FileNotFoundException("File not found", path); + } + + var extension = Path.GetExtension(path.AsSpan()); + if (extension.Equals(".svg", StringComparison.OrdinalIgnoreCase)) + { + var svg = new SKSvg(); + try + { + svg.Load(path); + return new ImageDimensions(Convert.ToInt32(svg.Picture.CullRect.Width), Convert.ToInt32(svg.Picture.CullRect.Height)); + } + catch (FormatException skiaColorException) + { + // This exception is known to be thrown on vector images that define custom styles + // Skia SVG is not able to handle that and as the repository is quite stale and has not received updates we just catch them + _logger.LogDebug(skiaColorException, "There was a issue loading the requested svg file"); + return default; + } + } + + using var codec = SKCodec.Create(path, out SKCodecResult result); + switch (result) + { + case SKCodecResult.Success: + var info = codec.Info; + return new ImageDimensions(info.Width, info.Height); + case SKCodecResult.Unimplemented: + _logger.LogDebug("Image format not supported: {FilePath}", path); + return default; + default: + _logger.LogError("Unable to determine image dimensions for {FilePath}: {SkCodecResult}", path, result); + return default; + } + } + + /// <inheritdoc /> + /// <exception cref="ArgumentNullException">The path is null.</exception> + /// <exception cref="FileNotFoundException">The path is not valid.</exception> + /// <exception cref="SkiaCodecException">The file at the specified path could not be used to generate a codec.</exception> + public string GetImageBlurHash(int xComp, int yComp, string path) + { + ArgumentException.ThrowIfNullOrEmpty(path); + + var extension = Path.GetExtension(path.AsSpan()).TrimStart('.'); + if (!SupportedInputFormats.Contains(extension, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogDebug("Unable to compute blur hash due to unsupported format: {ImagePath}", path); + return string.Empty; + } + + // Any larger than 128x128 is too slow and there's no visually discernible difference + return BlurHashEncoder.Encode(xComp, yComp, path, 128, 128); + } + + private bool RequiresSpecialCharacterHack(string path) + { + for (int i = 0; i < path.Length; i++) + { + if (char.GetUnicodeCategory(path[i]) == UnicodeCategory.OtherLetter) + { + return true; + } + } + + return path.HasDiacritics(); + } + + private string NormalizePath(string path) + { + if (!RequiresSpecialCharacterHack(path)) + { + return path; + } + + var tempPath = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + Path.GetExtension(path)); + var directory = Path.GetDirectoryName(tempPath) ?? throw new ResourceNotFoundException($"Provided path ({tempPath}) is not valid."); + Directory.CreateDirectory(directory); + File.Copy(path, tempPath, true); + + return tempPath; + } + + private static SKEncodedOrigin GetSKEncodedOrigin(ImageOrientation? orientation) + { + if (!orientation.HasValue) + { + return SKEncodedOrigin.TopLeft; + } + + return orientation.Value switch + { + ImageOrientation.TopRight => SKEncodedOrigin.TopRight, + ImageOrientation.RightTop => SKEncodedOrigin.RightTop, + ImageOrientation.RightBottom => SKEncodedOrigin.RightBottom, + ImageOrientation.LeftTop => SKEncodedOrigin.LeftTop, + ImageOrientation.LeftBottom => SKEncodedOrigin.LeftBottom, + ImageOrientation.BottomRight => SKEncodedOrigin.BottomRight, + ImageOrientation.BottomLeft => SKEncodedOrigin.BottomLeft, + _ => SKEncodedOrigin.TopLeft + }; + } + + /// <summary> + /// Decode an image. + /// </summary> + /// <param name="path">The filepath of the image to decode.</param> + /// <param name="forceCleanBitmap">Whether to force clean the bitmap.</param> + /// <param name="orientation">The orientation of the image.</param> + /// <param name="origin">The detected origin of the image.</param> + /// <returns>The resulting bitmap of the image.</returns> + internal SKBitmap? Decode(string path, bool forceCleanBitmap, ImageOrientation? orientation, out SKEncodedOrigin origin) + { + if (!File.Exists(path)) + { + throw new FileNotFoundException("File not found", path); + } + + var requiresTransparencyHack = _transparentImageTypes.Contains(Path.GetExtension(path)); + + if (requiresTransparencyHack || forceCleanBitmap) + { + using SKCodec codec = SKCodec.Create(NormalizePath(path), out SKCodecResult res); + if (res != SKCodecResult.Success) + { + origin = GetSKEncodedOrigin(orientation); + return null; + } + + // create the bitmap + var bitmap = new SKBitmap(codec.Info.Width, codec.Info.Height, !requiresTransparencyHack); + + // decode + _ = codec.GetPixels(bitmap.Info, bitmap.GetPixels()); + + origin = codec.EncodedOrigin; + + return bitmap; + } + + var resultBitmap = SKBitmap.Decode(NormalizePath(path)); + + if (resultBitmap is null) + { + return Decode(path, true, orientation, out origin); + } + + // If we have to resize these they often end up distorted + if (resultBitmap.ColorType == SKColorType.Gray8) + { + using (resultBitmap) + { + return Decode(path, true, orientation, out origin); + } + } + + origin = SKEncodedOrigin.TopLeft; + return resultBitmap; + } + + private SKBitmap? GetBitmap(string path, bool autoOrient, ImageOrientation? orientation) + { + if (autoOrient) + { + var bitmap = Decode(path, true, orientation, out var origin); + + if (bitmap is not null && origin != SKEncodedOrigin.TopLeft) + { + using (bitmap) + { + return OrientImage(bitmap, origin); + } + } + + return bitmap; + } + + return Decode(path, false, orientation, out _); + } + + private SKBitmap OrientImage(SKBitmap bitmap, SKEncodedOrigin origin) + { + var needsFlip = origin == SKEncodedOrigin.LeftBottom + || origin == SKEncodedOrigin.LeftTop + || origin == SKEncodedOrigin.RightBottom + || origin == SKEncodedOrigin.RightTop; + var rotated = needsFlip + ? new SKBitmap(bitmap.Height, bitmap.Width) + : new SKBitmap(bitmap.Width, bitmap.Height); + using var surface = new SKCanvas(rotated); + var midX = (float)rotated.Width / 2; + var midY = (float)rotated.Height / 2; + + switch (origin) + { + case SKEncodedOrigin.TopRight: + surface.Scale(-1, 1, midX, midY); + break; + case SKEncodedOrigin.BottomRight: + surface.RotateDegrees(180, midX, midY); + break; + case SKEncodedOrigin.BottomLeft: + surface.Scale(1, -1, midX, midY); + break; + case SKEncodedOrigin.LeftTop: + surface.Translate(0, -rotated.Height); + surface.Scale(1, -1, midX, midY); + surface.RotateDegrees(-90); + break; + case SKEncodedOrigin.RightTop: + surface.Translate(rotated.Width, 0); + surface.RotateDegrees(90); + break; + case SKEncodedOrigin.RightBottom: + surface.Translate(rotated.Width, 0); + surface.Scale(1, -1, midX, midY); + surface.RotateDegrees(90); + break; + case SKEncodedOrigin.LeftBottom: + surface.Translate(0, rotated.Height); + surface.RotateDegrees(-90); + break; + } + + surface.DrawBitmap(bitmap, 0, 0); + return rotated; + } + + /// <summary> + /// Resizes an image on the CPU, by utilizing a surface and canvas. + /// + /// The convolutional matrix kernel used in this resize function gives a (light) sharpening effect. + /// This technique is similar to effect that can be created using for example the [Convolution matrix filter in GIMP](https://docs.gimp.org/2.10/en/gimp-filter-convolution-matrix.html). + /// </summary> + /// <param name="source">The source bitmap.</param> + /// <param name="targetInfo">This specifies the target size and other information required to create the surface.</param> + /// <param name="isAntialias">This enables anti-aliasing on the SKPaint instance.</param> + /// <param name="isDither">This enables dithering on the SKPaint instance.</param> + /// <returns>The resized image.</returns> + internal static SKImage ResizeImage(SKBitmap source, SKImageInfo targetInfo, bool isAntialias = false, bool isDither = false) + { + using var surface = SKSurface.Create(targetInfo); + using var canvas = surface.Canvas; + using var paint = new SKPaint + { + FilterQuality = SKFilterQuality.High, + IsAntialias = isAntialias, + IsDither = isDither + }; + + var kernel = new float[9] + { + 0, -.1f, 0, + -.1f, 1.4f, -.1f, + 0, -.1f, 0, + }; + + var kernelSize = new SKSizeI(3, 3); + var kernelOffset = new SKPointI(1, 1); + + paint.ImageFilter = SKImageFilter.CreateMatrixConvolution( + kernelSize, + kernel, + 1f, + 0f, + kernelOffset, + SKShaderTileMode.Clamp, + true); + + canvas.DrawBitmap( + source, + SKRect.Create(0, 0, source.Width, source.Height), + SKRect.Create(0, 0, targetInfo.Width, targetInfo.Height), + paint); + + return surface.Snapshot(); + } + + /// <inheritdoc/> + public string EncodeImage(string inputPath, DateTime dateModified, string outputPath, bool autoOrient, ImageOrientation? orientation, int quality, ImageProcessingOptions options, ImageFormat outputFormat) + { + ArgumentException.ThrowIfNullOrEmpty(inputPath); + ArgumentException.ThrowIfNullOrEmpty(outputPath); + + var inputFormat = Path.GetExtension(inputPath.AsSpan()).TrimStart('.'); + if (!SupportedInputFormats.Contains(inputFormat, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogDebug("Unable to encode image due to unsupported format: {ImagePath}", inputPath); + return inputPath; + } + + var skiaOutputFormat = GetImageFormat(outputFormat); + + var hasBackgroundColor = !string.IsNullOrWhiteSpace(options.BackgroundColor); + var hasForegroundColor = !string.IsNullOrWhiteSpace(options.ForegroundLayer); + var blur = options.Blur ?? 0; + var hasIndicator = options.UnplayedCount.HasValue || !options.PercentPlayed.Equals(0); + + using var bitmap = GetBitmap(inputPath, autoOrient, orientation); + if (bitmap is null) + { + throw new InvalidDataException($"Skia unable to read image {inputPath}"); + } + + var originalImageSize = new ImageDimensions(bitmap.Width, bitmap.Height); + + if (options.HasDefaultOptions(inputPath, originalImageSize) && !autoOrient) + { + // Just spit out the original file if all the options are default + return inputPath; + } + + var newImageSize = ImageHelper.GetNewImageSize(options, originalImageSize); + + var width = newImageSize.Width; + var height = newImageSize.Height; + + // scale image (the FromImage creates a copy) + var imageInfo = new SKImageInfo(width, height, bitmap.ColorType, bitmap.AlphaType, bitmap.ColorSpace); + using var resizedBitmap = SKBitmap.FromImage(ResizeImage(bitmap, imageInfo)); + + // If all we're doing is resizing then we can stop now + if (!hasBackgroundColor && !hasForegroundColor && blur == 0 && !hasIndicator) + { + var outputDirectory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)); + Directory.CreateDirectory(outputDirectory); + using var outputStream = new SKFileWStream(outputPath); + using var pixmap = new SKPixmap(new SKImageInfo(width, height), resizedBitmap.GetPixels()); + resizedBitmap.Encode(outputStream, skiaOutputFormat, quality); + return outputPath; + } + + // create bitmap to use for canvas drawing used to draw into bitmap + using var saveBitmap = new SKBitmap(width, height); + using var canvas = new SKCanvas(saveBitmap); + // set background color if present + if (hasBackgroundColor) + { + canvas.Clear(SKColor.Parse(options.BackgroundColor)); + } + + // Add blur if option is present + if (blur > 0) + { + // create image from resized bitmap to apply blur + using var paint = new SKPaint(); + using var filter = SKImageFilter.CreateBlur(blur, blur); + paint.ImageFilter = filter; + canvas.DrawBitmap(resizedBitmap, SKRect.Create(width, height), paint); + } + else + { + // draw resized bitmap onto canvas + canvas.DrawBitmap(resizedBitmap, SKRect.Create(width, height)); + } + + // If foreground layer present then draw + if (hasForegroundColor) + { + if (!double.TryParse(options.ForegroundLayer, out double opacity)) + { + opacity = .4; + } + + canvas.DrawColor(new SKColor(0, 0, 0, (byte)((1 - opacity) * 0xFF)), SKBlendMode.SrcOver); + } + + if (hasIndicator) + { + DrawIndicator(canvas, width, height, options); + } + + var directory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)); + Directory.CreateDirectory(directory); + using (var outputStream = new SKFileWStream(outputPath)) + { + using var pixmap = new SKPixmap(new SKImageInfo(width, height), saveBitmap.GetPixels()); + pixmap.Encode(outputStream, skiaOutputFormat, quality); + } + + return outputPath; + } + + /// <inheritdoc/> + public void CreateImageCollage(ImageCollageOptions options, string? libraryName) + { + double ratio = (double)options.Width / options.Height; + + if (ratio >= 1.4) + { + new StripCollageBuilder(this).BuildThumbCollage(options.InputPaths, options.OutputPath, options.Width, options.Height, libraryName); + } + else if (ratio >= .9) + { + new StripCollageBuilder(this).BuildSquareCollage(options.InputPaths, options.OutputPath, options.Width, options.Height); + } + else + { + // TODO: Create Poster collage capability + new StripCollageBuilder(this).BuildSquareCollage(options.InputPaths, options.OutputPath, options.Width, options.Height); + } + } + + /// <inheritdoc /> + public void CreateSplashscreen(IReadOnlyList<string> posters, IReadOnlyList<string> backdrops) + { + var splashBuilder = new SplashscreenBuilder(this); + var outputPath = Path.Combine(_appPaths.DataPath, "splashscreen.png"); + splashBuilder.GenerateSplash(posters, backdrops, outputPath); + } + + private void DrawIndicator(SKCanvas canvas, int imageWidth, int imageHeight, ImageProcessingOptions options) + { + try + { + var currentImageSize = new ImageDimensions(imageWidth, imageHeight); + + if (options.UnplayedCount.HasValue) + { + UnplayedCountIndicator.DrawUnplayedCountIndicator(canvas, currentImageSize, options.UnplayedCount.Value); + } + + if (options.PercentPlayed > 0) + { + PercentPlayedDrawer.Process(canvas, currentImageSize, options.PercentPlayed); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error drawing indicator overlay"); + } + } +} diff --git a/src/Jellyfin.Drawing.Skia/SkiaException.cs b/src/Jellyfin.Drawing.Skia/SkiaException.cs new file mode 100644 index 000000000..d0e69d42c --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/SkiaException.cs @@ -0,0 +1,38 @@ +using System; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Represents errors that occur during interaction with Skia. +/// </summary> +public class SkiaException : Exception +{ + /// <summary> + /// Initializes a new instance of the <see cref="SkiaException"/> class. + /// </summary> + public SkiaException() + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="SkiaException"/> class with a specified error message. + /// </summary> + /// <param name="message">The message that describes the error.</param> + public SkiaException(string message) : base(message) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="SkiaException"/> class with a specified error message and a + /// reference to the inner exception that is the cause of this exception. + /// </summary> + /// <param name="message">The error message that explains the reason for the exception.</param> + /// <param name="innerException"> + /// The exception that is the cause of the current exception, or a null reference (Nothing in Visual Basic) if + /// no inner exception is specified. + /// </param> + public SkiaException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/Jellyfin.Drawing.Skia/SkiaHelper.cs b/src/Jellyfin.Drawing.Skia/SkiaHelper.cs new file mode 100644 index 000000000..00d224da9 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/SkiaHelper.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using SkiaSharp; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Class containing helper methods for working with SkiaSharp. +/// </summary> +public static class SkiaHelper +{ + /// <summary> + /// Gets the next valid image as a bitmap. + /// </summary> + /// <param name="skiaEncoder">The current skia encoder.</param> + /// <param name="paths">The list of image paths.</param> + /// <param name="currentIndex">The current checked index.</param> + /// <param name="newIndex">The new index.</param> + /// <returns>A valid bitmap, or null if no bitmap exists after <c>currentIndex</c>.</returns> + public static SKBitmap? GetNextValidImage(SkiaEncoder skiaEncoder, IReadOnlyList<string> paths, int currentIndex, out int newIndex) + { + var imagesTested = new Dictionary<int, int>(); + SKBitmap? bitmap = null; + + while (imagesTested.Count < paths.Count) + { + if (currentIndex >= paths.Count) + { + currentIndex = 0; + } + + bitmap = skiaEncoder.Decode(paths[currentIndex], false, null, out _); + + imagesTested[currentIndex] = 0; + + currentIndex++; + + if (bitmap is not null) + { + break; + } + } + + newIndex = currentIndex; + return bitmap; + } +} diff --git a/src/Jellyfin.Drawing.Skia/SplashscreenBuilder.cs b/src/Jellyfin.Drawing.Skia/SplashscreenBuilder.cs new file mode 100644 index 000000000..990556623 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/SplashscreenBuilder.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using SkiaSharp; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Used to build the splashscreen. +/// </summary> +public class SplashscreenBuilder +{ + private const int FinalWidth = 1920; + private const int FinalHeight = 1080; + // generated collage resolution should be higher than the final resolution + private const int WallWidth = FinalWidth * 3; + private const int WallHeight = FinalHeight * 2; + private const int Rows = 6; + private const int Spacing = 20; + + private readonly SkiaEncoder _skiaEncoder; + + /// <summary> + /// Initializes a new instance of the <see cref="SplashscreenBuilder"/> class. + /// </summary> + /// <param name="skiaEncoder">The SkiaEncoder.</param> + public SplashscreenBuilder(SkiaEncoder skiaEncoder) + { + _skiaEncoder = skiaEncoder; + } + + /// <summary> + /// Generate a splashscreen. + /// </summary> + /// <param name="posters">The poster paths.</param> + /// <param name="backdrops">The landscape paths.</param> + /// <param name="outputPath">The output path.</param> + public void GenerateSplash(IReadOnlyList<string> posters, IReadOnlyList<string> backdrops, string outputPath) + { + using var wall = GenerateCollage(posters, backdrops); + using var transformed = Transform3D(wall); + + using var outputStream = new SKFileWStream(outputPath); + using var pixmap = new SKPixmap(new SKImageInfo(FinalWidth, FinalHeight), transformed.GetPixels()); + pixmap.Encode(outputStream, StripCollageBuilder.GetEncodedFormat(outputPath), 90); + } + + /// <summary> + /// Generates a collage of posters and landscape pictures. + /// </summary> + /// <param name="posters">The poster paths.</param> + /// <param name="backdrops">The landscape paths.</param> + /// <returns>The created collage as a bitmap.</returns> + private SKBitmap GenerateCollage(IReadOnlyList<string> posters, IReadOnlyList<string> backdrops) + { + var posterIndex = 0; + var backdropIndex = 0; + + var bitmap = new SKBitmap(WallWidth, WallHeight); + using var canvas = new SKCanvas(bitmap); + canvas.Clear(SKColors.Black); + + int posterHeight = WallHeight / 6; + + for (int i = 0; i < Rows; i++) + { + int imageCounter = Random.Shared.Next(0, 5); + int currentWidthPos = i * 75; + int currentHeight = i * (posterHeight + Spacing); + + while (currentWidthPos < WallWidth) + { + SKBitmap? currentImage; + + switch (imageCounter) + { + case 0: + case 2: + case 3: + currentImage = SkiaHelper.GetNextValidImage(_skiaEncoder, posters, posterIndex, out int newPosterIndex); + posterIndex = newPosterIndex; + break; + default: + currentImage = SkiaHelper.GetNextValidImage(_skiaEncoder, backdrops, backdropIndex, out int newBackdropIndex); + backdropIndex = newBackdropIndex; + break; + } + + if (currentImage is null) + { + throw new ArgumentException("Not enough valid pictures provided to create a splashscreen!"); + } + + // resize to the same aspect as the original + var imageWidth = Math.Abs(posterHeight * currentImage.Width / currentImage.Height); + using var resizedBitmap = new SKBitmap(imageWidth, posterHeight); + currentImage.ScalePixels(resizedBitmap, SKFilterQuality.High); + + // draw on canvas + canvas.DrawBitmap(resizedBitmap, currentWidthPos, currentHeight); + + currentWidthPos += imageWidth + Spacing; + + currentImage.Dispose(); + + if (imageCounter >= 4) + { + imageCounter = 0; + } + else + { + imageCounter++; + } + } + } + + return bitmap; + } + + /// <summary> + /// Transform the collage in 3D space. + /// </summary> + /// <param name="input">The bitmap to transform.</param> + /// <returns>The transformed image.</returns> + private SKBitmap Transform3D(SKBitmap input) + { + var bitmap = new SKBitmap(FinalWidth, FinalHeight); + using var canvas = new SKCanvas(bitmap); + canvas.Clear(SKColors.Black); + var matrix = new SKMatrix + { + ScaleX = 0.324108899f, + ScaleY = 0.563934922f, + SkewX = -0.244337708f, + SkewY = 0.0377609022f, + TransX = 42.0407715f, + TransY = -198.104706f, + Persp0 = -9.08959337E-05f, + Persp1 = 6.85242048E-05f, + Persp2 = 0.988209724f + }; + + canvas.SetMatrix(matrix); + canvas.DrawBitmap(input, 0, 0); + + return bitmap; + } +} diff --git a/src/Jellyfin.Drawing.Skia/StripCollageBuilder.cs b/src/Jellyfin.Drawing.Skia/StripCollageBuilder.cs new file mode 100644 index 000000000..a8f80f7e2 --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/StripCollageBuilder.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; +using SkiaSharp; +using SkiaSharp.HarfBuzz; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Used to build collages of multiple images arranged in vertical strips. +/// </summary> +public partial class StripCollageBuilder +{ + private readonly SkiaEncoder _skiaEncoder; + + /// <summary> + /// Initializes a new instance of the <see cref="StripCollageBuilder"/> class. + /// </summary> + /// <param name="skiaEncoder">The encoder to use for building collages.</param> + public StripCollageBuilder(SkiaEncoder skiaEncoder) + { + _skiaEncoder = skiaEncoder; + } + + [GeneratedRegex(@"[^\p{IsCJKUnifiedIdeographs}\p{IsCJKUnifiedIdeographsExtensionA}\p{IsKatakana}\p{IsHiragana}\p{IsHangulSyllables}\p{IsHangulJamo}]")] + private static partial Regex NonCjkPatternRegex(); + + [GeneratedRegex(@"\p{IsArabic}|\p{IsArmenian}|\p{IsHebrew}|\p{IsSyriac}|\p{IsThaana}")] + private static partial Regex IsRtlTextRegex(); + + /// <summary> + /// Check which format an image has been encoded with using its filename extension. + /// </summary> + /// <param name="outputPath">The path to the image to get the format for.</param> + /// <returns>The image format.</returns> + public static SKEncodedImageFormat GetEncodedFormat(string outputPath) + { + ArgumentNullException.ThrowIfNull(outputPath); + + var ext = Path.GetExtension(outputPath); + + if (string.Equals(ext, ".jpg", StringComparison.OrdinalIgnoreCase) + || string.Equals(ext, ".jpeg", StringComparison.OrdinalIgnoreCase)) + { + return SKEncodedImageFormat.Jpeg; + } + + if (string.Equals(ext, ".webp", StringComparison.OrdinalIgnoreCase)) + { + return SKEncodedImageFormat.Webp; + } + + if (string.Equals(ext, ".gif", StringComparison.OrdinalIgnoreCase)) + { + return SKEncodedImageFormat.Gif; + } + + if (string.Equals(ext, ".bmp", StringComparison.OrdinalIgnoreCase)) + { + return SKEncodedImageFormat.Bmp; + } + + // default to png + return SKEncodedImageFormat.Png; + } + + /// <summary> + /// Create a square collage. + /// </summary> + /// <param name="paths">The paths of the images to use in the collage.</param> + /// <param name="outputPath">The path at which to place the resulting collage image.</param> + /// <param name="width">The desired width of the collage.</param> + /// <param name="height">The desired height of the collage.</param> + public void BuildSquareCollage(IReadOnlyList<string> paths, string outputPath, int width, int height) + { + using var bitmap = BuildSquareCollageBitmap(paths, width, height); + using var outputStream = new SKFileWStream(outputPath); + using var pixmap = new SKPixmap(new SKImageInfo(width, height), bitmap.GetPixels()); + pixmap.Encode(outputStream, GetEncodedFormat(outputPath), 90); + } + + /// <summary> + /// Create a thumb collage. + /// </summary> + /// <param name="paths">The paths of the images to use in the collage.</param> + /// <param name="outputPath">The path at which to place the resulting image.</param> + /// <param name="width">The desired width of the collage.</param> + /// <param name="height">The desired height of the collage.</param> + /// <param name="libraryName">The name of the library to draw on the collage.</param> + public void BuildThumbCollage(IReadOnlyList<string> paths, string outputPath, int width, int height, string? libraryName) + { + using var bitmap = BuildThumbCollageBitmap(paths, width, height, libraryName); + using var outputStream = new SKFileWStream(outputPath); + using var pixmap = new SKPixmap(new SKImageInfo(width, height), bitmap.GetPixels()); + pixmap.Encode(outputStream, GetEncodedFormat(outputPath), 90); + } + + private SKBitmap BuildThumbCollageBitmap(IReadOnlyList<string> paths, int width, int height, string? libraryName) + { + var bitmap = new SKBitmap(width, height); + + using var canvas = new SKCanvas(bitmap); + canvas.Clear(SKColors.Black); + + using var backdrop = SkiaHelper.GetNextValidImage(_skiaEncoder, paths, 0, out _); + if (backdrop is null) + { + return bitmap; + } + + // resize to the same aspect as the original + var backdropHeight = Math.Abs(width * backdrop.Height / backdrop.Width); + using var residedBackdrop = SkiaEncoder.ResizeImage(backdrop, new SKImageInfo(width, backdropHeight, backdrop.ColorType, backdrop.AlphaType, backdrop.ColorSpace)); + // draw the backdrop + canvas.DrawImage(residedBackdrop, 0, 0); + + // draw shadow rectangle + using var paintColor = new SKPaint + { + Color = SKColors.Black.WithAlpha(0x78), + Style = SKPaintStyle.Fill + }; + canvas.DrawRect(0, 0, width, height, paintColor); + + var typeFace = SKTypeface.FromFamilyName("sans-serif", SKFontStyleWeight.Bold, SKFontStyleWidth.Normal, SKFontStyleSlant.Upright); + + // use the system fallback to find a typeface for the given CJK character + var filteredName = NonCjkPatternRegex().Replace(libraryName ?? string.Empty, string.Empty); + if (!string.IsNullOrEmpty(filteredName)) + { + typeFace = SKFontManager.Default.MatchCharacter(null, SKFontStyleWeight.Bold, SKFontStyleWidth.Normal, SKFontStyleSlant.Upright, null, filteredName[0]); + } + + // draw library name + using var textPaint = new SKPaint + { + Color = SKColors.White, + Style = SKPaintStyle.Fill, + TextSize = 112, + TextAlign = SKTextAlign.Center, + Typeface = typeFace, + IsAntialias = true + }; + + // scale down text to 90% of the width if text is larger than 95% of the width + var textWidth = textPaint.MeasureText(libraryName); + if (textWidth > width * 0.95) + { + textPaint.TextSize = 0.9f * width * textPaint.TextSize / textWidth; + } + + if (string.IsNullOrWhiteSpace(libraryName)) + { + return bitmap; + } + + if (IsRtlTextRegex().IsMatch(libraryName)) + { + canvas.DrawShapedText(libraryName, width / 2f, (height / 2f) + (textPaint.FontMetrics.XHeight / 2), textPaint); + } + else + { + canvas.DrawText(libraryName, width / 2f, (height / 2f) + (textPaint.FontMetrics.XHeight / 2), textPaint); + } + + return bitmap; + } + + private SKBitmap BuildSquareCollageBitmap(IReadOnlyList<string> paths, int width, int height) + { + var bitmap = new SKBitmap(width, height); + var imageIndex = 0; + var cellWidth = width / 2; + var cellHeight = height / 2; + + using var canvas = new SKCanvas(bitmap); + for (var x = 0; x < 2; x++) + { + for (var y = 0; y < 2; y++) + { + using var currentBitmap = SkiaHelper.GetNextValidImage(_skiaEncoder, paths, imageIndex, out int newIndex); + imageIndex = newIndex; + + if (currentBitmap is null) + { + continue; + } + + // Scale image. The FromBitmap creates a copy + var imageInfo = new SKImageInfo(cellWidth, cellHeight, currentBitmap.ColorType, currentBitmap.AlphaType, currentBitmap.ColorSpace); + using var resizedBitmap = SKBitmap.FromImage(SkiaEncoder.ResizeImage(currentBitmap, imageInfo)); + + // draw this image into the strip at the next position + var xPos = x * cellWidth; + var yPos = y * cellHeight; + canvas.DrawBitmap(resizedBitmap, xPos, yPos); + } + } + + return bitmap; + } +} diff --git a/src/Jellyfin.Drawing.Skia/UnplayedCountIndicator.cs b/src/Jellyfin.Drawing.Skia/UnplayedCountIndicator.cs new file mode 100644 index 000000000..456b84b8c --- /dev/null +++ b/src/Jellyfin.Drawing.Skia/UnplayedCountIndicator.cs @@ -0,0 +1,63 @@ +using System.Globalization; +using MediaBrowser.Model.Drawing; +using SkiaSharp; + +namespace Jellyfin.Drawing.Skia; + +/// <summary> +/// Static helper class for drawing unplayed count indicators. +/// </summary> +public static class UnplayedCountIndicator +{ + /// <summary> + /// The x-offset used when drawing an unplayed count indicator. + /// </summary> + private const int OffsetFromTopRightCorner = 38; + + /// <summary> + /// Draw an unplayed count indicator in the top right corner of a canvas. + /// </summary> + /// <param name="canvas">The canvas to draw the indicator on.</param> + /// <param name="imageSize"> + /// The dimensions of the image to draw the indicator on. The width is used to determine the x-position of the + /// indicator. + /// </param> + /// <param name="count">The number to draw in the indicator.</param> + public static void DrawUnplayedCountIndicator(SKCanvas canvas, ImageDimensions imageSize, int count) + { + var x = imageSize.Width - OffsetFromTopRightCorner; + var text = count.ToString(CultureInfo.InvariantCulture); + + using var paint = new SKPaint + { + Color = SKColor.Parse("#CC00A4DC"), + Style = SKPaintStyle.Fill + }; + + canvas.DrawCircle(x, OffsetFromTopRightCorner, 20, paint); + + paint.Color = new SKColor(255, 255, 255, 255); + paint.TextSize = 24; + paint.IsAntialias = true; + + var y = OffsetFromTopRightCorner + 9; + + if (text.Length == 1) + { + x -= 7; + } + + if (text.Length == 2) + { + x -= 13; + } + else if (text.Length >= 3) + { + x -= 15; + y -= 2; + paint.TextSize = 18; + } + + canvas.DrawText(text, x, y, paint); + } +} diff --git a/src/Jellyfin.Drawing/ImageProcessor.cs b/src/Jellyfin.Drawing/ImageProcessor.cs new file mode 100644 index 000000000..4f16e294b --- /dev/null +++ b/src/Jellyfin.Drawing/ImageProcessor.cs @@ -0,0 +1,586 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Mime; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Entities; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Drawing; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Model.Drawing; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Net; +using Microsoft.Extensions.Logging; +using Photo = MediaBrowser.Controller.Entities.Photo; + +namespace Jellyfin.Drawing; + +/// <summary> +/// Class ImageProcessor. +/// </summary> +public sealed class ImageProcessor : IImageProcessor, IDisposable +{ + // Increment this when there's a change requiring caches to be invalidated + private const char Version = '3'; + + private static readonly HashSet<string> _transparentImageTypes + = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { ".png", ".webp", ".gif" }; + + private readonly ILogger<ImageProcessor> _logger; + private readonly IFileSystem _fileSystem; + private readonly IServerApplicationPaths _appPaths; + private readonly IImageEncoder _imageEncoder; + + private readonly SemaphoreSlim _parallelEncodingLimit; + + private bool _disposed; + + /// <summary> + /// Initializes a new instance of the <see cref="ImageProcessor"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="appPaths">The server application paths.</param> + /// <param name="fileSystem">The filesystem.</param> + /// <param name="imageEncoder">The image encoder.</param> + /// <param name="config">The configuration.</param> + public ImageProcessor( + ILogger<ImageProcessor> logger, + IServerApplicationPaths appPaths, + IFileSystem fileSystem, + IImageEncoder imageEncoder, + IServerConfigurationManager config) + { + _logger = logger; + _fileSystem = fileSystem; + _imageEncoder = imageEncoder; + _appPaths = appPaths; + + var semaphoreCount = config.Configuration.ParallelImageEncodingLimit; + if (semaphoreCount < 1) + { + semaphoreCount = 2 * Environment.ProcessorCount; + } + + _parallelEncodingLimit = new(semaphoreCount, semaphoreCount); + } + + private string ResizedImageCachePath => Path.Combine(_appPaths.ImageCachePath, "resized-images"); + + /// <inheritdoc /> + public IReadOnlyCollection<string> SupportedInputFormats => + new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + "tiff", + "tif", + "jpeg", + "jpg", + "png", + "aiff", + "cr2", + "crw", + "nef", + "orf", + "pef", + "arw", + "webp", + "gif", + "bmp", + "erf", + "raf", + "rw2", + "nrw", + "dng", + "ico", + "astc", + "ktx", + "pkm", + "wbmp" + }; + + /// <inheritdoc /> + public bool SupportsImageCollageCreation => _imageEncoder.SupportsImageCollageCreation; + + /// <inheritdoc /> + public async Task ProcessImage(ImageProcessingOptions options, Stream toStream) + { + var file = await ProcessImage(options).ConfigureAwait(false); + using var fileStream = AsyncFile.OpenRead(file.Path); + await fileStream.CopyToAsync(toStream).ConfigureAwait(false); + } + + /// <inheritdoc /> + public IReadOnlyCollection<ImageFormat> GetSupportedImageOutputFormats() + => _imageEncoder.SupportedOutputFormats; + + /// <inheritdoc /> + public bool SupportsTransparency(string path) + => _transparentImageTypes.Contains(Path.GetExtension(path)); + + /// <inheritdoc /> + public async Task<(string Path, string? MimeType, DateTime DateModified)> ProcessImage(ImageProcessingOptions options) + { + ItemImageInfo originalImage = options.Image; + BaseItem item = options.Item; + + string originalImagePath = originalImage.Path; + DateTime dateModified = originalImage.DateModified; + ImageDimensions? originalImageSize = null; + if (originalImage.Width > 0 && originalImage.Height > 0) + { + originalImageSize = new ImageDimensions(originalImage.Width, originalImage.Height); + } + + var mimeType = MimeTypes.GetMimeType(originalImagePath); + if (!_imageEncoder.SupportsImageEncoding) + { + return (originalImagePath, mimeType, dateModified); + } + + var supportedImageInfo = await GetSupportedImage(originalImagePath, dateModified).ConfigureAwait(false); + originalImagePath = supportedImageInfo.Path; + + // Original file doesn't exist, or original file is gif. + if (!File.Exists(originalImagePath) || string.Equals(mimeType, MediaTypeNames.Image.Gif, StringComparison.OrdinalIgnoreCase)) + { + return (originalImagePath, mimeType, dateModified); + } + + dateModified = supportedImageInfo.DateModified; + bool requiresTransparency = _transparentImageTypes.Contains(Path.GetExtension(originalImagePath)); + + bool autoOrient = false; + ImageOrientation? orientation = null; + if (item is Photo photo) + { + if (photo.Orientation.HasValue) + { + if (photo.Orientation.Value != ImageOrientation.TopLeft) + { + autoOrient = true; + orientation = photo.Orientation; + } + } + else + { + // Orientation unknown, so do it + autoOrient = true; + orientation = photo.Orientation; + } + } + + if (options.HasDefaultOptions(originalImagePath, originalImageSize) && (!autoOrient || !options.RequiresAutoOrientation)) + { + // Just spit out the original file if all the options are default + return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); + } + + int quality = options.Quality; + + ImageFormat outputFormat = GetOutputFormat(options.SupportedOutputFormats, requiresTransparency); + string cacheFilePath = GetCacheFilePath( + originalImagePath, + options.Width, + options.Height, + options.MaxWidth, + options.MaxHeight, + options.FillWidth, + options.FillHeight, + quality, + dateModified, + outputFormat, + options.PercentPlayed, + options.UnplayedCount, + options.Blur, + options.BackgroundColor, + options.ForegroundLayer); + + try + { + if (!File.Exists(cacheFilePath)) + { + // Limit number of parallel (more precisely: concurrent) image encodings to prevent a high memory usage + await _parallelEncodingLimit.WaitAsync().ConfigureAwait(false); + + string resultPath; + try + { + resultPath = _imageEncoder.EncodeImage(originalImagePath, dateModified, cacheFilePath, autoOrient, orientation, quality, options, outputFormat); + } + finally + { + _parallelEncodingLimit.Release(); + } + + if (string.Equals(resultPath, originalImagePath, StringComparison.OrdinalIgnoreCase)) + { + return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); + } + } + + return (cacheFilePath, GetMimeType(outputFormat, cacheFilePath), _fileSystem.GetLastWriteTimeUtc(cacheFilePath)); + } + catch (Exception ex) + { + // If it fails for whatever reason, return the original image + _logger.LogError(ex, "Error encoding image"); + return (originalImagePath, MimeTypes.GetMimeType(originalImagePath), dateModified); + } + } + + private ImageFormat GetOutputFormat(IReadOnlyCollection<ImageFormat> clientSupportedFormats, bool requiresTransparency) + { + var serverFormats = GetSupportedImageOutputFormats(); + + // Client doesn't care about format, so start with webp if supported + if (serverFormats.Contains(ImageFormat.Webp) && clientSupportedFormats.Contains(ImageFormat.Webp)) + { + return ImageFormat.Webp; + } + + // If transparency is needed and webp isn't supported, than png is the only option + if (requiresTransparency && clientSupportedFormats.Contains(ImageFormat.Png)) + { + return ImageFormat.Png; + } + + foreach (var format in clientSupportedFormats) + { + if (serverFormats.Contains(format)) + { + return format; + } + } + + // We should never actually get here + return ImageFormat.Jpg; + } + + private string GetMimeType(ImageFormat format, string path) + => format switch + { + ImageFormat.Bmp => MimeTypes.GetMimeType("i.bmp"), + ImageFormat.Gif => MimeTypes.GetMimeType("i.gif"), + ImageFormat.Jpg => MimeTypes.GetMimeType("i.jpg"), + ImageFormat.Png => MimeTypes.GetMimeType("i.png"), + ImageFormat.Webp => MimeTypes.GetMimeType("i.webp"), + _ => MimeTypes.GetMimeType(path) + }; + + /// <summary> + /// Gets the cache file path based on a set of parameters. + /// </summary> + private string GetCacheFilePath( + string originalPath, + int? width, + int? height, + int? maxWidth, + int? maxHeight, + int? fillWidth, + int? fillHeight, + int quality, + DateTime dateModified, + ImageFormat format, + double percentPlayed, + int? unwatchedCount, + int? blur, + string backgroundColor, + string foregroundLayer) + { + var filename = new StringBuilder(256); + filename.Append(originalPath); + + filename.Append(",quality="); + filename.Append(quality); + + filename.Append(",datemodified="); + filename.Append(dateModified.Ticks); + + filename.Append(",f="); + filename.Append(format); + + if (width.HasValue) + { + filename.Append(",width="); + filename.Append(width.Value); + } + + if (height.HasValue) + { + filename.Append(",height="); + filename.Append(height.Value); + } + + if (maxWidth.HasValue) + { + filename.Append(",maxwidth="); + filename.Append(maxWidth.Value); + } + + if (maxHeight.HasValue) + { + filename.Append(",maxheight="); + filename.Append(maxHeight.Value); + } + + if (fillWidth.HasValue) + { + filename.Append(",fillwidth="); + filename.Append(fillWidth.Value); + } + + if (fillHeight.HasValue) + { + filename.Append(",fillheight="); + filename.Append(fillHeight.Value); + } + + if (percentPlayed > 0) + { + filename.Append(",p="); + filename.Append(percentPlayed); + } + + if (unwatchedCount.HasValue) + { + filename.Append(",p="); + filename.Append(unwatchedCount.Value); + } + + if (blur.HasValue) + { + filename.Append(",blur="); + filename.Append(blur.Value); + } + + if (!string.IsNullOrEmpty(backgroundColor)) + { + filename.Append(",b="); + filename.Append(backgroundColor); + } + + if (!string.IsNullOrEmpty(foregroundLayer)) + { + filename.Append(",fl="); + filename.Append(foregroundLayer); + } + + filename.Append(",v="); + filename.Append(Version); + + return GetCachePath(ResizedImageCachePath, filename.ToString(), "." + format.ToString().ToLowerInvariant()); + } + + /// <inheritdoc /> + public ImageDimensions GetImageDimensions(BaseItem item, ItemImageInfo info) + { + int width = info.Width; + int height = info.Height; + + if (height > 0 && width > 0) + { + return new ImageDimensions(width, height); + } + + string path = info.Path; + _logger.LogDebug("Getting image size for item {ItemType} {Path}", item.GetType().Name, path); + + ImageDimensions size = GetImageDimensions(path); + info.Width = size.Width; + info.Height = size.Height; + + return size; + } + + /// <inheritdoc /> + public ImageDimensions GetImageDimensions(string path) + => _imageEncoder.GetImageSize(path); + + /// <inheritdoc /> + public string GetImageBlurHash(string path) + { + var size = GetImageDimensions(path); + return GetImageBlurHash(path, size); + } + + /// <inheritdoc /> + public string GetImageBlurHash(string path, ImageDimensions imageDimensions) + { + if (imageDimensions.Width <= 0 || imageDimensions.Height <= 0) + { + return string.Empty; + } + + // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. + // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. + // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components + float xCompF = MathF.Sqrt(16.0f * imageDimensions.Width / imageDimensions.Height); + float yCompF = xCompF * imageDimensions.Height / imageDimensions.Width; + + int xComp = Math.Min((int)xCompF + 1, 9); + int yComp = Math.Min((int)yCompF + 1, 9); + + return _imageEncoder.GetImageBlurHash(xComp, yComp, path); + } + + /// <inheritdoc /> + public string GetImageCacheTag(BaseItem item, ItemImageInfo image) + => (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture); + + /// <inheritdoc /> + public string? GetImageCacheTag(BaseItem item, ChapterInfo chapter) + { + if (chapter.ImagePath is null) + { + return null; + } + + return GetImageCacheTag(item, new ItemImageInfo + { + Path = chapter.ImagePath, + Type = ImageType.Chapter, + DateModified = chapter.ImageDateModified + }); + } + + /// <inheritdoc /> + public string? GetImageCacheTag(User user) + { + if (user.ProfileImage is null) + { + return null; + } + + return (user.ProfileImage.Path + user.ProfileImage.LastModified.Ticks).GetMD5() + .ToString("N", CultureInfo.InvariantCulture); + } + + private Task<(string Path, DateTime DateModified)> GetSupportedImage(string originalImagePath, DateTime dateModified) + { + var inputFormat = Path.GetExtension(originalImagePath.AsSpan()).TrimStart('.').ToString(); + + // These are just jpg files renamed as tbn + if (string.Equals(inputFormat, "tbn", StringComparison.OrdinalIgnoreCase)) + { + return Task.FromResult((originalImagePath, dateModified)); + } + + // TODO _mediaEncoder.ConvertImage is not implemented + // if (!_imageEncoder.SupportedInputFormats.Contains(inputFormat)) + // { + // try + // { + // string filename = (originalImagePath + dateModified.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5().ToString("N", CultureInfo.InvariantCulture); + // + // string cacheExtension = _mediaEncoder.SupportsEncoder("libwebp") ? ".webp" : ".png"; + // var outputPath = Path.Combine(_appPaths.ImageCachePath, "converted-images", filename + cacheExtension); + // + // var file = _fileSystem.GetFileInfo(outputPath); + // if (!file.Exists) + // { + // await _mediaEncoder.ConvertImage(originalImagePath, outputPath).ConfigureAwait(false); + // dateModified = _fileSystem.GetLastWriteTimeUtc(outputPath); + // } + // else + // { + // dateModified = file.LastWriteTimeUtc; + // } + // + // originalImagePath = outputPath; + // } + // catch (Exception ex) + // { + // _logger.LogError(ex, "Image conversion failed for {Path}", originalImagePath); + // } + // } + + return Task.FromResult((originalImagePath, dateModified)); + } + + /// <summary> + /// Gets the cache path. + /// </summary> + /// <param name="path">The path.</param> + /// <param name="uniqueName">Name of the unique.</param> + /// <param name="fileExtension">The file extension.</param> + /// <returns>System.String.</returns> + /// <exception cref="ArgumentNullException"> + /// path + /// or + /// uniqueName + /// or + /// fileExtension. + /// </exception> + public string GetCachePath(string path, string uniqueName, string fileExtension) + { + ArgumentException.ThrowIfNullOrEmpty(path); + ArgumentException.ThrowIfNullOrEmpty(uniqueName); + ArgumentException.ThrowIfNullOrEmpty(fileExtension); + + var filename = uniqueName.GetMD5() + fileExtension; + + return GetCachePath(path, filename); + } + + /// <summary> + /// Gets the cache path. + /// </summary> + /// <param name="path">The path.</param> + /// <param name="filename">The filename.</param> + /// <returns>System.String.</returns> + /// <exception cref="ArgumentNullException"> + /// path + /// or + /// filename. + /// </exception> + public string GetCachePath(ReadOnlySpan<char> path, ReadOnlySpan<char> filename) + { + if (path.IsEmpty) + { + throw new ArgumentException("Path can't be empty.", nameof(path)); + } + + if (filename.IsEmpty) + { + throw new ArgumentException("Filename can't be empty.", nameof(filename)); + } + + var prefix = filename.Slice(0, 1); + + return Path.Join(path, prefix, filename); + } + + /// <inheritdoc /> + public void CreateImageCollage(ImageCollageOptions options, string? libraryName) + { + _logger.LogInformation("Creating image collage and saving to {Path}", options.OutputPath); + + _imageEncoder.CreateImageCollage(options, libraryName); + + _logger.LogInformation("Completed creation of image collage and saved to {Path}", options.OutputPath); + } + + /// <inheritdoc /> + public void Dispose() + { + if (_disposed) + { + return; + } + + if (_imageEncoder is IDisposable disposable) + { + disposable.Dispose(); + } + + _parallelEncodingLimit?.Dispose(); + + _disposed = true; + } +} diff --git a/src/Jellyfin.Drawing/Jellyfin.Drawing.csproj b/src/Jellyfin.Drawing/Jellyfin.Drawing.csproj new file mode 100644 index 000000000..e0963ac34 --- /dev/null +++ b/src/Jellyfin.Drawing/Jellyfin.Drawing.csproj @@ -0,0 +1,35 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <!-- ProjectGuid is only included as a requirement for SonarQube analysis --> + <PropertyGroup> + <ProjectGuid>{08FFF49B-F175-4807-A2B5-73B0EBD9F716}</ProjectGuid> + </PropertyGroup> + + <PropertyGroup> + <TargetFramework>net7.0</TargetFramework> + <GenerateAssemblyInfo>false</GenerateAssemblyInfo> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="..\..\MediaBrowser.Model\MediaBrowser.Model.csproj" /> + <ProjectReference Include="..\..\MediaBrowser.Controller\MediaBrowser.Controller.csproj" /> + <ProjectReference Include="..\..\MediaBrowser.Common\MediaBrowser.Common.csproj" /> + </ItemGroup> + + <ItemGroup> + <Compile Include="..\..\SharedVersion.cs" /> + </ItemGroup> + + <!-- Code analysers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + <PackageReference Include="SerilogAnalyzer" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" PrivateAssets="All" /> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.Drawing/NullImageEncoder.cs b/src/Jellyfin.Drawing/NullImageEncoder.cs new file mode 100644 index 000000000..171128bed --- /dev/null +++ b/src/Jellyfin.Drawing/NullImageEncoder.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using MediaBrowser.Controller.Drawing; +using MediaBrowser.Model.Drawing; + +namespace Jellyfin.Drawing; + +/// <summary> +/// A fallback implementation of <see cref="IImageEncoder" />. +/// </summary> +public class NullImageEncoder : IImageEncoder +{ + /// <inheritdoc /> + public IReadOnlyCollection<string> SupportedInputFormats + => new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "png", "jpeg", "jpg" }; + + /// <inheritdoc /> + public IReadOnlyCollection<ImageFormat> SupportedOutputFormats + => new HashSet<ImageFormat>() { ImageFormat.Jpg, ImageFormat.Png }; + + /// <inheritdoc /> + public string Name => "Null Image Encoder"; + + /// <inheritdoc /> + public bool SupportsImageCollageCreation => false; + + /// <inheritdoc /> + public bool SupportsImageEncoding => false; + + /// <inheritdoc /> + public ImageDimensions GetImageSize(string path) + => throw new NotImplementedException(); + + /// <inheritdoc /> + public string EncodeImage(string inputPath, DateTime dateModified, string outputPath, bool autoOrient, ImageOrientation? orientation, int quality, ImageProcessingOptions options, ImageFormat outputFormat) + { + throw new NotImplementedException(); + } + + /// <inheritdoc /> + public void CreateImageCollage(ImageCollageOptions options, string? libraryName) + { + throw new NotImplementedException(); + } + + /// <inheritdoc /> + public void CreateSplashscreen(IReadOnlyList<string> posters, IReadOnlyList<string> backdrops) + { + throw new NotImplementedException(); + } + + /// <inheritdoc /> + public string GetImageBlurHash(int xComp, int yComp, string path) + { + throw new NotImplementedException(); + } +} diff --git a/src/Jellyfin.Drawing/Properties/AssemblyInfo.cs b/src/Jellyfin.Drawing/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..3851bf924 --- /dev/null +++ b/src/Jellyfin.Drawing/Properties/AssemblyInfo.cs @@ -0,0 +1,30 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Jellyfin.Drawing")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Jellyfin Project")] +[assembly: AssemblyProduct("Jellyfin Server")] +[assembly: AssemblyCopyright("Copyright © 2019 Jellyfin Contributors. Code released under the GNU General Public License")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("87b6f14e-16d8-4a58-a553-fd9945e47458")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// diff --git a/src/Jellyfin.Extensions/AlphanumericComparator.cs b/src/Jellyfin.Extensions/AlphanumericComparator.cs new file mode 100644 index 000000000..299e2f94a --- /dev/null +++ b/src/Jellyfin.Extensions/AlphanumericComparator.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Alphanumeric <see cref="IComparer{T}" />. + /// </summary> + public class AlphanumericComparator : IComparer<string?> + { + /// <summary> + /// Compares two objects and returns a value indicating whether one is less than, equal to, or greater than the other. + /// </summary> + /// <param name="s1">The first object to compare.</param> + /// <param name="s2">The second object to compare.</param> + /// <returns>A signed integer that indicates the relative values of <c>x</c> and <c>y</c>.</returns> + public static int CompareValues(string? s1, string? s2) + { + if (s1 is null && s2 is null) + { + return 0; + } + + if (s1 is null) + { + return -1; + } + + if (s2 is null) + { + return 1; + } + + int len1 = s1.Length; + int len2 = s2.Length; + + // Early return for empty strings + if (len1 == 0 && len2 == 0) + { + return 0; + } + + if (len1 == 0) + { + return -1; + } + + if (len2 == 0) + { + return 1; + } + + int pos1 = 0; + int pos2 = 0; + + do + { + int start1 = pos1; + int start2 = pos2; + + bool isNum1 = char.IsDigit(s1[pos1++]); + bool isNum2 = char.IsDigit(s2[pos2++]); + + while (pos1 < len1 && char.IsDigit(s1[pos1]) == isNum1) + { + pos1++; + } + + while (pos2 < len2 && char.IsDigit(s2[pos2]) == isNum2) + { + pos2++; + } + + var span1 = s1.AsSpan(start1, pos1 - start1); + var span2 = s2.AsSpan(start2, pos2 - start2); + + if (isNum1 && isNum2) + { + // Trim leading zeros so we can compare the length + // of the strings to find the largest number + span1 = span1.TrimStart('0'); + span2 = span2.TrimStart('0'); + var span1Len = span1.Length; + var span2Len = span2.Length; + if (span1Len < span2Len) + { + return -1; + } + + if (span1Len > span2Len) + { + return 1; + } + } + + int result = span1.CompareTo(span2, StringComparison.InvariantCulture); + if (result != 0) + { + return result; + } + } while (pos1 < len1 && pos2 < len2); + + return len1 - len2; + } + + /// <inheritdoc /> + public int Compare(string? x, string? y) + { + return CompareValues(x, y); + } + } +} diff --git a/src/Jellyfin.Extensions/CopyToExtensions.cs b/src/Jellyfin.Extensions/CopyToExtensions.cs new file mode 100644 index 000000000..72d37b5b6 --- /dev/null +++ b/src/Jellyfin.Extensions/CopyToExtensions.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Provides <c>CopyTo</c> extensions methods for <see cref="IReadOnlyList{T}" />. + /// </summary> + public static class CopyToExtensions + { + /// <summary> + /// Copies all the elements of the current collection to the specified list + /// starting at the specified destination array index. The index is specified as a 32-bit integer. + /// </summary> + /// <param name="source">The current collection that is the source of the elements.</param> + /// <param name="destination">The list that is the destination of the elements copied from the current collection.</param> + /// <param name="index">A 32-bit integer that represents the index in <c>destination</c> at which copying begins.</param> + /// <typeparam name="T">The type of the array.</typeparam> + public static void CopyTo<T>(this IReadOnlyList<T> source, IList<T> destination, int index = 0) + { + for (int i = 0; i < source.Count; i++) + { + destination[index + i] = source[i]; + } + } + } +} diff --git a/src/Jellyfin.Extensions/DictionaryExtensions.cs b/src/Jellyfin.Extensions/DictionaryExtensions.cs new file mode 100644 index 000000000..5bb828d01 --- /dev/null +++ b/src/Jellyfin.Extensions/DictionaryExtensions.cs @@ -0,0 +1,64 @@ +using System.Collections.Generic; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Static extensions for the <see cref="IReadOnlyDictionary{TKey,TValue}"/> interface. + /// </summary> + public static class DictionaryExtensions + { + /// <summary> + /// Gets a string from a string dictionary, checking all keys sequentially, + /// stopping at the first key that returns a result that's neither null nor blank. + /// </summary> + /// <param name="dictionary">The dictionary.</param> + /// <param name="key1">The first checked key.</param> + /// <returns>System.String.</returns> + public static string? GetFirstNotNullNorWhiteSpaceValue(this IReadOnlyDictionary<string, string> dictionary, string key1) + { + return dictionary.GetFirstNotNullNorWhiteSpaceValue(key1, string.Empty, string.Empty); + } + + /// <summary> + /// Gets a string from a string dictionary, checking all keys sequentially, + /// stopping at the first key that returns a result that's neither null nor blank. + /// </summary> + /// <param name="dictionary">The dictionary.</param> + /// <param name="key1">The first checked key.</param> + /// <param name="key2">The second checked key.</param> + /// <returns>System.String.</returns> + public static string? GetFirstNotNullNorWhiteSpaceValue(this IReadOnlyDictionary<string, string> dictionary, string key1, string key2) + { + return dictionary.GetFirstNotNullNorWhiteSpaceValue(key1, key2, string.Empty); + } + + /// <summary> + /// Gets a string from a string dictionary, checking all keys sequentially, + /// stopping at the first key that returns a result that's neither null nor blank. + /// </summary> + /// <param name="dictionary">The dictionary.</param> + /// <param name="key1">The first checked key.</param> + /// <param name="key2">The second checked key.</param> + /// <param name="key3">The third checked key.</param> + /// <returns>System.String.</returns> + public static string? GetFirstNotNullNorWhiteSpaceValue(this IReadOnlyDictionary<string, string> dictionary, string key1, string key2, string key3) + { + if (dictionary.TryGetValue(key1, out var val) && !string.IsNullOrWhiteSpace(val)) + { + return val; + } + + if (!string.IsNullOrEmpty(key2) && dictionary.TryGetValue(key2, out val) && !string.IsNullOrWhiteSpace(val)) + { + return val; + } + + if (!string.IsNullOrEmpty(key3) && dictionary.TryGetValue(key3, out val) && !string.IsNullOrWhiteSpace(val)) + { + return val; + } + + return null; + } + } +} diff --git a/src/Jellyfin.Extensions/EnumerableExtensions.cs b/src/Jellyfin.Extensions/EnumerableExtensions.cs new file mode 100644 index 000000000..fd46358a4 --- /dev/null +++ b/src/Jellyfin.Extensions/EnumerableExtensions.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; + +namespace Jellyfin.Extensions; + +/// <summary> +/// Static extensions for the <see cref="IEnumerable{T}"/> interface. +/// </summary> +public static class EnumerableExtensions +{ + /// <summary> + /// Determines whether the value is contained in the source collection. + /// </summary> + /// <param name="source">An instance of the <see cref="IEnumerable{String}"/> interface.</param> + /// <param name="value">The value to look for in the collection.</param> + /// <param name="stringComparison">The string comparison.</param> + /// <returns>A value indicating whether the value is contained in the collection.</returns> + /// <exception cref="ArgumentNullException">The source is null.</exception> + public static bool Contains(this IEnumerable<string> source, ReadOnlySpan<char> value, StringComparison stringComparison) + { + ArgumentNullException.ThrowIfNull(source); + + if (source is IList<string> list) + { + int len = list.Count; + for (int i = 0; i < len; i++) + { + if (value.Equals(list[i], stringComparison)) + { + return true; + } + } + + return false; + } + + foreach (string element in source) + { + if (value.Equals(element, stringComparison)) + { + return true; + } + } + + return false; + } + + /// <summary> + /// Gets an IEnumerable from a single item. + /// </summary> + /// <param name="item">The item to return.</param> + /// <typeparam name="T">The type of item.</typeparam> + /// <returns>The IEnumerable{T}.</returns> + public static IEnumerable<T> SingleItemAsEnumerable<T>(this T item) + { + yield return item; + } +} diff --git a/src/Jellyfin.Extensions/Jellyfin.Extensions.csproj b/src/Jellyfin.Extensions/Jellyfin.Extensions.csproj new file mode 100644 index 000000000..4f80aa941 --- /dev/null +++ b/src/Jellyfin.Extensions/Jellyfin.Extensions.csproj @@ -0,0 +1,46 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net7.0</TargetFramework> + <GenerateAssemblyInfo>false</GenerateAssemblyInfo> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + <PublishRepositoryUrl>true</PublishRepositoryUrl> + <EmbedUntrackedSources>true</EmbedUntrackedSources> + <IncludeSymbols>true</IncludeSymbols> + <SymbolPackageFormat>snupkg</SymbolPackageFormat> + </PropertyGroup> + + <PropertyGroup> + <Authors>Jellyfin Contributors</Authors> + <PackageId>Jellyfin.Extensions</PackageId> + <VersionPrefix>10.9.0</VersionPrefix> + <RepositoryUrl>https://github.com/jellyfin/jellyfin</RepositoryUrl> + <PackageLicenseExpression>GPL-3.0-only</PackageLicenseExpression> + </PropertyGroup> + + <PropertyGroup Condition=" '$(Stability)'=='Unstable'"> + <!-- Include all symbols in the main nupkg until Azure Artifact Feed starts supporting ingesting NuGet symbol packages. --> + <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder> + </PropertyGroup> + + <ItemGroup> + <Compile Include="../../SharedVersion.cs" /> + </ItemGroup> + + + <ItemGroup> + <PackageReference Include="Diacritics" /> + </ItemGroup> + + <!-- Code Analyzers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + <PackageReference Include="SerilogAnalyzer" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" PrivateAssets="All" /> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonBoolNumberConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonBoolNumberConverter.cs new file mode 100644 index 000000000..c2543cf7c --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonBoolNumberConverter.cs @@ -0,0 +1,30 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts a number to a boolean. + /// This is needed for HDHomerun. + /// </summary> + public class JsonBoolNumberConverter : JsonConverter<bool> + { + /// <inheritdoc /> + public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Number) + { + return Convert.ToBoolean(reader.GetInt32()); + } + + return reader.GetBoolean(); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) + { + writer.WriteBooleanValue(value); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonBoolStringConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonBoolStringConverter.cs new file mode 100644 index 000000000..6895eadb8 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonBoolStringConverter.cs @@ -0,0 +1,33 @@ +using System; +using System.Buffers; +using System.Buffers.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters; + +/// <summary> +/// Converts a string to a boolean. +/// This is needed for FFprobe. +/// </summary> +public class JsonBoolStringConverter : JsonConverter<bool> +{ + /// <inheritdoc /> + public override bool Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + ReadOnlySpan<byte> utf8Span = reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan; + if (Utf8Parser.TryParse(utf8Span, out bool val, out _, 'l')) + { + return val; + } + } + + return reader.GetBoolean(); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, bool value, JsonSerializerOptions options) + => writer.WriteBooleanValue(value); +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverter.cs new file mode 100644 index 000000000..0d0cc2d06 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverter.cs @@ -0,0 +1,19 @@ +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Convert comma delimited string to array of type. + /// </summary> + /// <typeparam name="T">Type to convert to.</typeparam> + public sealed class JsonCommaDelimitedArrayConverter<T> : JsonDelimitedArrayConverter<T> + { + /// <summary> + /// Initializes a new instance of the <see cref="JsonCommaDelimitedArrayConverter{T}"/> class. + /// </summary> + public JsonCommaDelimitedArrayConverter() : base() + { + } + + /// <inheritdoc /> + protected override char Delimiter => ','; + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverterFactory.cs b/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverterFactory.cs new file mode 100644 index 000000000..cc9311a24 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonCommaDelimitedArrayConverterFactory.cs @@ -0,0 +1,28 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Json comma delimited array converter factory. + /// </summary> + /// <remarks> + /// This must be applied as an attribute, adding to the JsonConverter list causes stack overflow. + /// </remarks> + public class JsonCommaDelimitedArrayConverterFactory : JsonConverterFactory + { + /// <inheritdoc /> + public override bool CanConvert(Type typeToConvert) + { + return true; + } + + /// <inheritdoc /> + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var structType = typeToConvert.GetElementType() ?? typeToConvert.GenericTypeArguments[0]; + return (JsonConverter?)Activator.CreateInstance(typeof(JsonCommaDelimitedArrayConverter<>).MakeGenericType(structType)); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonDateTimeConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonDateTimeConverter.cs new file mode 100644 index 000000000..8ae080b15 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonDateTimeConverter.cs @@ -0,0 +1,34 @@ +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Legacy DateTime converter. + /// Milliseconds aren't output if zero by default. + /// </summary> + public class JsonDateTimeConverter : JsonConverter<DateTime> + { + /// <inheritdoc /> + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return reader.GetDateTime(); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + if (value.Millisecond == 0) + { + // Remaining ticks value will be 0, manually format. + writer.WriteStringValue(value.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fffffffZ", CultureInfo.InvariantCulture)); + } + else + { + writer.WriteStringValue(value); + } + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonDelimitedArrayConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonDelimitedArrayConverter.cs new file mode 100644 index 000000000..17096c017 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonDelimitedArrayConverter.cs @@ -0,0 +1,81 @@ +using System; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Convert delimited string to array of type. + /// </summary> + /// <typeparam name="T">Type to convert to.</typeparam> + public abstract class JsonDelimitedArrayConverter<T> : JsonConverter<T[]> + { + private readonly TypeConverter _typeConverter; + + /// <summary> + /// Initializes a new instance of the <see cref="JsonDelimitedArrayConverter{T}"/> class. + /// </summary> + protected JsonDelimitedArrayConverter() + { + _typeConverter = TypeDescriptor.GetConverter(typeof(T)); + } + + /// <summary> + /// Gets the array delimiter. + /// </summary> + protected virtual char Delimiter { get; } + + /// <inheritdoc /> + public override T[]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + // null got handled higher up the call stack + var stringEntries = reader.GetString()!.Split(Delimiter, StringSplitOptions.RemoveEmptyEntries); + if (stringEntries.Length == 0) + { + return Array.Empty<T>(); + } + + var parsedValues = new object[stringEntries.Length]; + var convertedCount = 0; + for (var i = 0; i < stringEntries.Length; i++) + { + try + { + parsedValues[i] = _typeConverter.ConvertFromInvariantString(stringEntries[i].Trim()) ?? throw new FormatException(); + convertedCount++; + } + catch (FormatException) + { + // TODO log when upgraded to .Net6 + // https://github.com/dotnet/runtime/issues/42975 + // _logger.LogDebug(e, "Error converting value."); + } + } + + var typedValues = new T[convertedCount]; + var typedValueIndex = 0; + for (var i = 0; i < stringEntries.Length; i++) + { + if (parsedValues[i] is not null) + { + typedValues.SetValue(parsedValues[i], typedValueIndex); + typedValueIndex++; + } + } + + return typedValues; + } + + return JsonSerializer.Deserialize<T[]>(ref reader, options); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, T[]? value, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverter.cs new file mode 100644 index 000000000..4fa91fa5e --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverter.cs @@ -0,0 +1,36 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters; + +/// <summary> +/// Enum flag to json array converter. +/// </summary> +/// <typeparam name="T">The type of enum.</typeparam> +public class JsonFlagEnumConverter<T> : JsonConverter<T> + where T : struct, Enum +{ + private static readonly T[] _enumValues = Enum.GetValues<T>(); + + /// <inheritdoc /> + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + writer.WriteStartArray(); + foreach (var enumValue in _enumValues) + { + if (value.HasFlag(enumValue)) + { + writer.WriteStringValue(enumValue.ToString()); + } + } + + writer.WriteEndArray(); + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverterFactory.cs b/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverterFactory.cs new file mode 100644 index 000000000..b74caf345 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonFlagEnumConverterFactory.cs @@ -0,0 +1,24 @@ +using System; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters; + +/// <summary> +/// Json flag enum converter factory. +/// </summary> +public class JsonFlagEnumConverterFactory : JsonConverterFactory +{ + /// <inheritdoc /> + public override bool CanConvert(Type typeToConvert) + { + return typeToConvert.IsEnum && typeToConvert.IsDefined(typeof(FlagsAttribute)); + } + + /// <inheritdoc /> + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + return (JsonConverter?)Activator.CreateInstance(typeof(JsonFlagEnumConverter<>).MakeGenericType(typeToConvert)); + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonGuidConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonGuidConverter.cs new file mode 100644 index 000000000..ea6d141cb --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonGuidConverter.cs @@ -0,0 +1,30 @@ +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts a GUID object or value to/from JSON. + /// </summary> + public class JsonGuidConverter : JsonConverter<Guid> + { + /// <inheritdoc /> + public override Guid Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => reader.TokenType == JsonTokenType.Null + ? Guid.Empty + : ReadInternal(ref reader); + + // TODO: optimize by parsing the UTF8 bytes instead of converting to string first + internal static Guid ReadInternal(ref Utf8JsonReader reader) + => Guid.Parse(reader.GetString()!); // null got handled higher up the call stack + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, Guid value, JsonSerializerOptions options) + => WriteInternal(writer, value); + + internal static void WriteInternal(Utf8JsonWriter writer, Guid value) + => writer.WriteStringValue(value.ToString("N", CultureInfo.InvariantCulture)); + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonLowerCaseConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonLowerCaseConverter.cs new file mode 100644 index 000000000..cd582ced6 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonLowerCaseConverter.cs @@ -0,0 +1,25 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts an object to a lowercase string. + /// </summary> + /// <typeparam name="T">The object type.</typeparam> + public class JsonLowerCaseConverter<T> : JsonConverter<T> + { + /// <inheritdoc /> + public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return JsonSerializer.Deserialize<T>(ref reader, options); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + writer.WriteStringValue(value?.ToString()?.ToLowerInvariant()); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonNullableGuidConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonNullableGuidConverter.cs new file mode 100644 index 000000000..656e3c3da --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonNullableGuidConverter.cs @@ -0,0 +1,31 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts a GUID object or value to/from JSON. + /// </summary> + public class JsonNullableGuidConverter : JsonConverter<Guid?> + { + /// <inheritdoc /> + public override Guid? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => JsonGuidConverter.ReadInternal(ref reader); + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, Guid? value, JsonSerializerOptions options) + { + // null got handled higher up the call stack + var val = value!.Value; + if (val.Equals(default)) + { + writer.WriteNullValue(); + } + else + { + JsonGuidConverter.WriteInternal(writer, val); + } + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverter.cs new file mode 100644 index 000000000..28437023f --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverter.cs @@ -0,0 +1,33 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts a nullable struct or value to/from JSON. + /// Required - some clients send an empty string. + /// </summary> + /// <typeparam name="TStruct">The struct type.</typeparam> + public class JsonNullableStructConverter<TStruct> : JsonConverter<TStruct?> + where TStruct : struct + { + /// <inheritdoc /> + public override TStruct? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Token is empty string. + if (reader.TokenType == JsonTokenType.String + && ((reader.HasValueSequence && reader.ValueSequence.IsEmpty) + || (!reader.HasValueSequence && reader.ValueSpan.IsEmpty))) + { + return null; + } + + return JsonSerializer.Deserialize<TStruct>(ref reader, options); + } + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, TStruct? value, JsonSerializerOptions options) + => JsonSerializer.Serialize(writer, value!.Value, options); // null got handled higher up the call stack + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverterFactory.cs b/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverterFactory.cs new file mode 100644 index 000000000..e7749589a --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonNullableStructConverterFactory.cs @@ -0,0 +1,27 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Json nullable struct converter factory. + /// </summary> + public class JsonNullableStructConverterFactory : JsonConverterFactory + { + /// <inheritdoc /> + public override bool CanConvert(Type typeToConvert) + { + return typeToConvert.IsGenericType + && typeToConvert.GetGenericTypeDefinition() == typeof(Nullable<>) + && typeToConvert.GenericTypeArguments[0].IsValueType; + } + + /// <inheritdoc /> + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var structType = typeToConvert.GenericTypeArguments[0]; + return (JsonConverter?)Activator.CreateInstance(typeof(JsonNullableStructConverter<>).MakeGenericType(structType)); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverter.cs new file mode 100644 index 000000000..6e59fe464 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverter.cs @@ -0,0 +1,19 @@ +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Convert Pipe delimited string to array of type. + /// </summary> + /// <typeparam name="T">Type to convert to.</typeparam> + public sealed class JsonPipeDelimitedArrayConverter<T> : JsonDelimitedArrayConverter<T> + { + /// <summary> + /// Initializes a new instance of the <see cref="JsonPipeDelimitedArrayConverter{T}"/> class. + /// </summary> + public JsonPipeDelimitedArrayConverter() : base() + { + } + + /// <inheritdoc /> + protected override char Delimiter => '|'; + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverterFactory.cs b/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverterFactory.cs new file mode 100644 index 000000000..579674f18 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonPipeDelimitedArrayConverterFactory.cs @@ -0,0 +1,28 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Json Pipe delimited array converter factory. + /// </summary> + /// <remarks> + /// This must be applied as an attribute, adding to the JsonConverter list causes stack overflow. + /// </remarks> + public class JsonPipeDelimitedArrayConverterFactory : JsonConverterFactory + { + /// <inheritdoc /> + public override bool CanConvert(Type typeToConvert) + { + return true; + } + + /// <inheritdoc /> + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var structType = typeToConvert.GetElementType() ?? typeToConvert.GenericTypeArguments[0]; + return (JsonConverter?)Activator.CreateInstance(typeof(JsonPipeDelimitedArrayConverter<>).MakeGenericType(structType)); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonStringConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonStringConverter.cs new file mode 100644 index 000000000..36b36c9b4 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonStringConverter.cs @@ -0,0 +1,30 @@ +using System; +using System.Buffers; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converter to allow the serializer to read strings. + /// </summary> + public class JsonStringConverter : JsonConverter<string?> + { + /// <inheritdoc /> + public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => reader.TokenType == JsonTokenType.String ? reader.GetString() : GetRawValue(reader); + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, string? value, JsonSerializerOptions options) + => writer.WriteStringValue(value); + + private static string GetRawValue(Utf8JsonReader reader) + { + var utf8Bytes = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; + return Encoding.UTF8.GetString(utf8Bytes); + } + } +} diff --git a/src/Jellyfin.Extensions/Json/Converters/JsonVersionConverter.cs b/src/Jellyfin.Extensions/Json/Converters/JsonVersionConverter.cs new file mode 100644 index 000000000..51ffec1cb --- /dev/null +++ b/src/Jellyfin.Extensions/Json/Converters/JsonVersionConverter.cs @@ -0,0 +1,23 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Jellyfin.Extensions.Json.Converters +{ + /// <summary> + /// Converts a Version object or value to/from JSON. + /// </summary> + /// <remarks> + /// Required to send <see cref="Version"/> as a string instead of an object. + /// </remarks> + public class JsonVersionConverter : JsonConverter<Version> + { + /// <inheritdoc /> + public override Version Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => new Version(reader.GetString()!); // Will throw ArgumentNullException on null + + /// <inheritdoc /> + public override void Write(Utf8JsonWriter writer, Version value, JsonSerializerOptions options) + => writer.WriteStringValue(value.ToString()); + } +} diff --git a/src/Jellyfin.Extensions/Json/JsonDefaults.cs b/src/Jellyfin.Extensions/Json/JsonDefaults.cs new file mode 100644 index 000000000..4d56ca615 --- /dev/null +++ b/src/Jellyfin.Extensions/Json/JsonDefaults.cs @@ -0,0 +1,90 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Jellyfin.Extensions.Json.Converters; + +namespace Jellyfin.Extensions.Json +{ + /// <summary> + /// Helper class for having compatible JSON throughout the codebase. + /// </summary> + public static class JsonDefaults + { + /// <summary> + /// Pascal case json profile media type. + /// </summary> + public const string PascalCaseMediaType = "application/json; profile=\"PascalCase\""; + + /// <summary> + /// Camel case json profile media type. + /// </summary> + public const string CamelCaseMediaType = "application/json; profile=\"CamelCase\""; + + /// <summary> + /// When changing these options, update + /// Jellyfin.Server/Extensions/ApiServiceCollectionExtensions.cs + /// -> AddJellyfinApi + /// -> AddJsonOptions. + /// </summary> + private static readonly JsonSerializerOptions _jsonSerializerOptions = new() + { + ReadCommentHandling = JsonCommentHandling.Disallow, + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + NumberHandling = JsonNumberHandling.AllowReadingFromString, + Converters = + { + new JsonGuidConverter(), + new JsonNullableGuidConverter(), + new JsonVersionConverter(), + new JsonFlagEnumConverterFactory(), + new JsonStringEnumConverter(), + new JsonNullableStructConverterFactory(), + new JsonDateTimeConverter(), + new JsonStringConverter() + } + }; + + private static readonly JsonSerializerOptions _pascalCaseJsonSerializerOptions = new(_jsonSerializerOptions) + { + PropertyNamingPolicy = null + }; + + private static readonly JsonSerializerOptions _camelCaseJsonSerializerOptions = new(_jsonSerializerOptions) + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + /// <summary> + /// Gets the default <see cref="JsonSerializerOptions" /> options. + /// </summary> + /// <remarks> + /// The return value must not be modified. + /// If the defaults must be modified the author must use the copy constructor. + /// </remarks> + /// <returns>The default <see cref="JsonSerializerOptions" /> options.</returns> + public static JsonSerializerOptions Options + => _jsonSerializerOptions; + + /// <summary> + /// Gets camelCase json options. + /// </summary> + /// <remarks> + /// The return value must not be modified. + /// If the defaults must be modified the author must use the copy constructor. + /// </remarks> + /// <returns>The camelCase <see cref="JsonSerializerOptions" /> options.</returns> + public static JsonSerializerOptions CamelCaseOptions + => _camelCaseJsonSerializerOptions; + + /// <summary> + /// Gets PascalCase json options. + /// </summary> + /// <remarks> + /// The return value must not be modified. + /// If the defaults must be modified the author must use the copy constructor. + /// </remarks> + /// <returns>The PascalCase <see cref="JsonSerializerOptions" /> options.</returns> + public static JsonSerializerOptions PascalCaseOptions + => _pascalCaseJsonSerializerOptions; + } +} diff --git a/src/Jellyfin.Extensions/ReadOnlyListExtension.cs b/src/Jellyfin.Extensions/ReadOnlyListExtension.cs new file mode 100644 index 000000000..ba99bb534 --- /dev/null +++ b/src/Jellyfin.Extensions/ReadOnlyListExtension.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Static extensions for the <see cref="IReadOnlyList{T}"/> interface. + /// </summary> + public static class ReadOnlyListExtension + { + /// <summary> + /// Finds the index of the desired item. + /// </summary> + /// <param name="source">The source list.</param> + /// <param name="value">The value to fine.</param> + /// <typeparam name="T">The type of item to find.</typeparam> + /// <returns>Index if found, else -1.</returns> + public static int IndexOf<T>(this IReadOnlyList<T> source, T value) + { + if (source is IList<T> list) + { + return list.IndexOf(value); + } + + for (int i = 0; i < source.Count; i++) + { + if (Equals(value, source[i])) + { + return i; + } + } + + return -1; + } + + /// <summary> + /// Finds the index of the predicate. + /// </summary> + /// <param name="source">The source list.</param> + /// <param name="match">The value to find.</param> + /// <typeparam name="T">The type of item to find.</typeparam> + /// <returns>Index if found, else -1.</returns> + public static int FindIndex<T>(this IReadOnlyList<T> source, Predicate<T> match) + { + if (source is List<T> list) + { + return list.FindIndex(match); + } + + for (int i = 0; i < source.Count; i++) + { + if (match(source[i])) + { + return i; + } + } + + return -1; + } + + /// <summary> + /// Get the first or default item from a list. + /// </summary> + /// <param name="source">The source list.</param> + /// <typeparam name="T">The type of item.</typeparam> + /// <returns>The first item or default if list is empty.</returns> + public static T? FirstOrDefault<T>(this IReadOnlyList<T>? source) + { + if (source is null || source.Count == 0) + { + return default; + } + + return source[0]; + } + } +} diff --git a/src/Jellyfin.Extensions/ShuffleExtensions.cs b/src/Jellyfin.Extensions/ShuffleExtensions.cs new file mode 100644 index 000000000..33c492053 --- /dev/null +++ b/src/Jellyfin.Extensions/ShuffleExtensions.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Provides <c>Shuffle</c> extensions methods for <see cref="IList{T}" />. + /// </summary> + public static class ShuffleExtensions + { + /// <summary> + /// Shuffles the items in a list. + /// </summary> + /// <param name="list">The list that should get shuffled.</param> + /// <typeparam name="T">The type.</typeparam> + public static void Shuffle<T>(this IList<T> list) + { + list.Shuffle(Random.Shared); + } + + /// <summary> + /// Shuffles the items in a list. + /// </summary> + /// <param name="list">The list that should get shuffled.</param> + /// <param name="rng">The random number generator to use.</param> + /// <typeparam name="T">The type.</typeparam> + public static void Shuffle<T>(this IList<T> list, Random rng) + { + int n = list.Count; + while (n > 1) + { + int k = rng.Next(n--); + T value = list[k]; + list[k] = list[n]; + list[n] = value; + } + } + } +} diff --git a/src/Jellyfin.Extensions/SplitStringExtensions.cs b/src/Jellyfin.Extensions/SplitStringExtensions.cs new file mode 100644 index 000000000..a4dc9fc6b --- /dev/null +++ b/src/Jellyfin.Extensions/SplitStringExtensions.cs @@ -0,0 +1,115 @@ +/* +MIT License + +Copyright (c) 2019 Gérald Barré + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + */ + +// TODO: remove when analyzer is fixed: https://github.com/dotnet/roslyn-analyzers/issues/5158 +#pragma warning disable CA1034 // Nested types should not be visible + +using System; +using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Extension class for splitting lines without unnecessary allocations. + /// </summary> + public static class SplitStringExtensions + { + /// <summary> + /// Creates a new string split enumerator. + /// </summary> + /// <param name="str">The string to split.</param> + /// <param name="separator">The separator to split on.</param> + /// <returns>The enumerator struct.</returns> + [Pure] + public static Enumerator SpanSplit(this string str, char separator) => new(str.AsSpan(), separator); + + /// <summary> + /// Creates a new span split enumerator. + /// </summary> + /// <param name="str">The span to split.</param> + /// <param name="separator">The separator to split on.</param> + /// <returns>The enumerator struct.</returns> + [Pure] + public static Enumerator Split(this ReadOnlySpan<char> str, char separator) => new(str, separator); + + /// <summary> + /// Provides an enumerator for the substrings separated by the separator. + /// </summary> + [StructLayout(LayoutKind.Auto)] + public ref struct Enumerator + { + private readonly char _separator; + private ReadOnlySpan<char> _str; + + /// <summary> + /// Initializes a new instance of the <see cref="Enumerator"/> struct. + /// </summary> + /// <param name="str">The span to split.</param> + /// <param name="separator">The separator to split on.</param> + public Enumerator(ReadOnlySpan<char> str, char separator) + { + _str = str; + _separator = separator; + Current = default; + } + + /// <summary> + /// Gets a reference to the item at the current position of the enumerator. + /// </summary> + public ReadOnlySpan<char> Current { get; private set; } + + /// <summary> + /// Returns <c>this</c>. + /// </summary> + /// <returns><c>this</c>.</returns> + public readonly Enumerator GetEnumerator() => this; + + /// <summary> + /// Advances the enumerator to the next item. + /// </summary> + /// <returns><c>true</c> if there is a next element; otherwise <c>false</c>.</returns> + public bool MoveNext() + { + if (_str.Length == 0) + { + return false; + } + + var span = _str; + var index = span.IndexOf(_separator); + if (index == -1) + { + _str = ReadOnlySpan<char>.Empty; + Current = span; + return true; + } + + Current = span.Slice(0, index); + _str = span[(index + 1)..]; + return true; + } + } + } +} diff --git a/src/Jellyfin.Extensions/StreamExtensions.cs b/src/Jellyfin.Extensions/StreamExtensions.cs new file mode 100644 index 000000000..182996852 --- /dev/null +++ b/src/Jellyfin.Extensions/StreamExtensions.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Class BaseExtensions. + /// </summary> + public static class StreamExtensions + { + /// <summary> + /// Reads all lines in the <see cref="Stream" />. + /// </summary> + /// <param name="stream">The <see cref="Stream" /> to read from.</param> + /// <returns>All lines in the stream.</returns> + public static string[] ReadAllLines(this Stream stream) + => ReadAllLines(stream, Encoding.UTF8); + + /// <summary> + /// Reads all lines in the <see cref="Stream" />. + /// </summary> + /// <param name="stream">The <see cref="Stream" /> to read from.</param> + /// <param name="encoding">The character encoding to use.</param> + /// <returns>All lines in the stream.</returns> + public static string[] ReadAllLines(this Stream stream, Encoding encoding) + { + using StreamReader reader = new StreamReader(stream, encoding); + return ReadAllLines(reader).ToArray(); + } + + /// <summary> + /// Reads all lines in the <see cref="TextReader" />. + /// </summary> + /// <param name="reader">The <see cref="TextReader" /> to read from.</param> + /// <returns>All lines in the stream.</returns> + public static IEnumerable<string> ReadAllLines(this TextReader reader) + { + string? line; + while ((line = reader.ReadLine()) is not null) + { + yield return line; + } + } + + /// <summary> + /// Reads all lines in the <see cref="TextReader" />. + /// </summary> + /// <param name="reader">The <see cref="TextReader" /> to read from.</param> + /// <returns>All lines in the stream.</returns> + public static async IAsyncEnumerable<string> ReadAllLinesAsync(this TextReader reader) + { + string? line; + while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) + { + yield return line; + } + } + } +} diff --git a/src/Jellyfin.Extensions/StringBuilderExtensions.cs b/src/Jellyfin.Extensions/StringBuilderExtensions.cs new file mode 100644 index 000000000..02ff7cc1f --- /dev/null +++ b/src/Jellyfin.Extensions/StringBuilderExtensions.cs @@ -0,0 +1,35 @@ +using System.Collections.Generic; +using System.Text; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Extension methods for the <see cref="StringBuilder"/> class. + /// </summary> + public static class StringBuilderExtensions + { + /// <summary> + /// Concatenates and appends the members of a collection in single quotes using the specified delimiter. + /// </summary> + /// <param name="builder">The string builder.</param> + /// <param name="delimiter">The character delimiter.</param> + /// <param name="values">The collection of strings to concatenate.</param> + /// <returns>The updated string builder.</returns> + public static StringBuilder AppendJoinInSingleQuotes(this StringBuilder builder, char delimiter, IReadOnlyList<string> values) + { + var len = values.Count; + for (var i = 0; i < len; i++) + { + builder.Append('\'') + .Append(values[i]) + .Append('\'') + .Append(delimiter); + } + + // remove last , + builder.Length--; + + return builder; + } + } +} diff --git a/src/Jellyfin.Extensions/StringExtensions.cs b/src/Jellyfin.Extensions/StringExtensions.cs new file mode 100644 index 000000000..fd8f7e59a --- /dev/null +++ b/src/Jellyfin.Extensions/StringExtensions.cs @@ -0,0 +1,90 @@ +using System; +using System.Text.RegularExpressions; + +namespace Jellyfin.Extensions +{ + /// <summary> + /// Provides extensions methods for <see cref="string" />. + /// </summary> + public static partial class StringExtensions + { + // Matches non-conforming unicode chars + // https://mnaoumov.wordpress.com/2014/06/14/stripping-invalid-characters-from-utf-16-strings/ + + [GeneratedRegex("([\ud800-\udbff](?![\udc00-\udfff]))|((?<![\ud800-\udbff])[\udc00-\udfff])|(�)")] + private static partial Regex NonConformingUnicodeRegex(); + + /// <summary> + /// Removes the diacritics character from the strings. + /// </summary> + /// <param name="text">The string to act on.</param> + /// <returns>The string without diacritics character.</returns> + public static string RemoveDiacritics(this string text) + => Diacritics.Extensions.StringExtensions.RemoveDiacritics( + NonConformingUnicodeRegex().Replace(text, string.Empty)); + + /// <summary> + /// Checks whether or not the specified string has diacritics in it. + /// </summary> + /// <param name="text">The string to check.</param> + /// <returns>True if the string has diacritics, false otherwise.</returns> + public static bool HasDiacritics(this string text) + => Diacritics.Extensions.StringExtensions.HasDiacritics(text) + || NonConformingUnicodeRegex().IsMatch(text); + + /// <summary> + /// Counts the number of occurrences of [needle] in the string. + /// </summary> + /// <param name="value">The haystack to search in.</param> + /// <param name="needle">The character to search for.</param> + /// <returns>The number of occurrences of the [needle] character.</returns> + public static int Count(this ReadOnlySpan<char> value, char needle) + { + var count = 0; + var length = value.Length; + for (var i = 0; i < length; i++) + { + if (value[i] == needle) + { + count++; + } + } + + return count; + } + + /// <summary> + /// Returns the part on the left of the <c>needle</c>. + /// </summary> + /// <param name="haystack">The string to seek.</param> + /// <param name="needle">The needle to find.</param> + /// <returns>The part left of the <paramref name="needle" />.</returns> + public static ReadOnlySpan<char> LeftPart(this ReadOnlySpan<char> haystack, char needle) + { + var pos = haystack.IndexOf(needle); + return pos == -1 ? haystack : haystack[..pos]; + } + + /// <summary> + /// Returns the part on the right of the <c>needle</c>. + /// </summary> + /// <param name="haystack">The string to seek.</param> + /// <param name="needle">The needle to find.</param> + /// <returns>The part right of the <paramref name="needle" />.</returns> + public static ReadOnlySpan<char> RightPart(this ReadOnlySpan<char> haystack, char needle) + { + var pos = haystack.LastIndexOf(needle); + if (pos == -1) + { + return haystack; + } + + if (pos == haystack.Length - 1) + { + return ReadOnlySpan<char>.Empty; + } + + return haystack[(pos + 1)..]; + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs new file mode 100644 index 000000000..360a96290 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Cache/CacheDecorator.cs @@ -0,0 +1,96 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Text.Json; +using Jellyfin.Extensions.Json; +using Jellyfin.MediaEncoding.Hls.Extractors; +using Jellyfin.MediaEncoding.Keyframes; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Extensions; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.MediaEncoding.Hls.Cache; + +/// <inheritdoc /> +public class CacheDecorator : IKeyframeExtractor +{ + private readonly IKeyframeExtractor _keyframeExtractor; + private readonly ILogger<CacheDecorator> _logger; + private readonly string _keyframeExtractorName; + private static readonly JsonSerializerOptions _jsonOptions = JsonDefaults.Options; + private readonly string _keyframeCachePath; + + /// <summary> + /// Initializes a new instance of the <see cref="CacheDecorator"/> class. + /// </summary> + /// <param name="applicationPaths">An instance of the <see cref="IApplicationPaths"/> interface.</param> + /// <param name="keyframeExtractor">An instance of the <see cref="IKeyframeExtractor"/> interface.</param> + /// <param name="logger">An instance of the <see cref="ILogger{CacheDecorator}"/> interface.</param> + public CacheDecorator(IApplicationPaths applicationPaths, IKeyframeExtractor keyframeExtractor, ILogger<CacheDecorator> logger) + { + ArgumentNullException.ThrowIfNull(applicationPaths); + ArgumentNullException.ThrowIfNull(keyframeExtractor); + + _keyframeExtractor = keyframeExtractor; + _logger = logger; + _keyframeExtractorName = keyframeExtractor.GetType().Name; + // TODO make the dir configurable + _keyframeCachePath = Path.Combine(applicationPaths.DataPath, "keyframes"); + } + + /// <inheritdoc /> + public bool IsMetadataBased => _keyframeExtractor.IsMetadataBased; + + /// <inheritdoc /> + public bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + keyframeData = null; + var cachePath = GetCachePath(_keyframeCachePath, filePath); + if (TryReadFromCache(cachePath, out var cachedResult)) + { + keyframeData = cachedResult; + return true; + } + + if (!_keyframeExtractor.TryExtractKeyframes(filePath, out var result)) + { + _logger.LogDebug("Failed to extract keyframes using {ExtractorName}", _keyframeExtractorName); + return false; + } + + _logger.LogDebug("Successfully extracted keyframes using {ExtractorName}", _keyframeExtractorName); + keyframeData = result; + SaveToCache(cachePath, keyframeData); + return true; + } + + private static void SaveToCache(string cachePath, KeyframeData keyframeData) + { + var json = JsonSerializer.Serialize(keyframeData, _jsonOptions); + Directory.CreateDirectory(Path.GetDirectoryName(cachePath) ?? throw new ArgumentException($"Provided path ({cachePath}) is not valid.", nameof(cachePath))); + File.WriteAllText(cachePath, json); + } + + private static string GetCachePath(string keyframeCachePath, string filePath) + { + var lastWriteTimeUtc = File.GetLastWriteTimeUtc(filePath); + ReadOnlySpan<char> filename = (filePath + "_" + lastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).GetMD5() + ".json"; + var prefix = filename[..1]; + + return Path.Join(keyframeCachePath, prefix, filename); + } + + private static bool TryReadFromCache(string cachePath, [NotNullWhen(true)] out KeyframeData? cachedResult) + { + if (File.Exists(cachePath)) + { + var bytes = File.ReadAllBytes(cachePath); + cachedResult = JsonSerializer.Deserialize<KeyframeData>(bytes, _jsonOptions); + return cachedResult is not null; + } + + cachedResult = null; + return false; + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs b/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs new file mode 100644 index 000000000..8ed4edcea --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Extensions/MediaEncodingHlsServiceCollectionExtensions.cs @@ -0,0 +1,36 @@ +using System; +using Jellyfin.MediaEncoding.Hls.Cache; +using Jellyfin.MediaEncoding.Hls.Extractors; +using Jellyfin.MediaEncoding.Hls.Playlist; +using Microsoft.Extensions.DependencyInjection; + +namespace Jellyfin.MediaEncoding.Hls.Extensions; + +/// <summary> +/// Extensions for the <see cref="IServiceCollection"/> interface. +/// </summary> +public static class MediaEncodingHlsServiceCollectionExtensions +{ + /// <summary> + /// Adds the hls playlist generators to the <see cref="IServiceCollection"/>. + /// </summary> + /// <param name="serviceCollection">An instance of the <see cref="IServiceCollection"/> interface.</param> + /// <returns>The updated service collection.</returns> + public static IServiceCollection AddHlsPlaylistGenerator(this IServiceCollection serviceCollection) + { + serviceCollection.AddSingletonWithDecorator(typeof(FfProbeKeyframeExtractor)); + serviceCollection.AddSingletonWithDecorator(typeof(MatroskaKeyframeExtractor)); + serviceCollection.AddSingleton<IDynamicHlsPlaylistGenerator, DynamicHlsPlaylistGenerator>(); + return serviceCollection; + } + + private static void AddSingletonWithDecorator(this IServiceCollection serviceCollection, Type type) + { + serviceCollection.AddSingleton<IKeyframeExtractor>(serviceProvider => + { + var extractor = ActivatorUtilities.CreateInstance(serviceProvider, type); + var decorator = ActivatorUtilities.CreateInstance<CacheDecorator>(serviceProvider, extractor); + return decorator; + }); + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs new file mode 100644 index 000000000..f86599a23 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Extractors/FfProbeKeyframeExtractor.cs @@ -0,0 +1,58 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using Emby.Naming.Common; +using Jellyfin.Extensions; +using Jellyfin.MediaEncoding.Keyframes; +using MediaBrowser.Controller.MediaEncoding; +using Microsoft.Extensions.Logging; +using Extractor = Jellyfin.MediaEncoding.Keyframes.FfProbe.FfProbeKeyframeExtractor; + +namespace Jellyfin.MediaEncoding.Hls.Extractors; + +/// <inheritdoc /> +public class FfProbeKeyframeExtractor : IKeyframeExtractor +{ + private readonly IMediaEncoder _mediaEncoder; + private readonly NamingOptions _namingOptions; + private readonly ILogger<FfProbeKeyframeExtractor> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="FfProbeKeyframeExtractor"/> class. + /// </summary> + /// <param name="mediaEncoder">An instance of the <see cref="IMediaEncoder"/> interface.</param> + /// <param name="namingOptions">An instance of <see cref="NamingOptions"/>.</param> + /// <param name="logger">An instance of the <see cref="ILogger{FfprobeKeyframeExtractor}"/> interface.</param> + public FfProbeKeyframeExtractor(IMediaEncoder mediaEncoder, NamingOptions namingOptions, ILogger<FfProbeKeyframeExtractor> logger) + { + _mediaEncoder = mediaEncoder; + _namingOptions = namingOptions; + _logger = logger; + } + + /// <inheritdoc /> + public bool IsMetadataBased => false; + + /// <inheritdoc /> + public bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + if (!_namingOptions.VideoFileExtensions.Contains(Path.GetExtension(filePath.AsSpan()), StringComparison.OrdinalIgnoreCase)) + { + keyframeData = null; + return false; + } + + try + { + keyframeData = Extractor.GetKeyframeData(_mediaEncoder.ProbePath, filePath); + return keyframeData.KeyframeTicks.Count > 0; + } + catch (Exception ex) + { + _logger.LogError(ex, "Extracting keyframes from {FilePath} using ffprobe failed", filePath); + } + + keyframeData = null; + return false; + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Extractors/IKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Hls/Extractors/IKeyframeExtractor.cs new file mode 100644 index 000000000..083e93de1 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Extractors/IKeyframeExtractor.cs @@ -0,0 +1,23 @@ +using System.Diagnostics.CodeAnalysis; +using Jellyfin.MediaEncoding.Keyframes; + +namespace Jellyfin.MediaEncoding.Hls.Extractors; + +/// <summary> +/// Keyframe extractor. +/// </summary> +public interface IKeyframeExtractor +{ + /// <summary> + /// Gets a value indicating whether the extractor is based on container metadata. + /// </summary> + bool IsMetadataBased { get; } + + /// <summary> + /// Attempt to extract keyframes. + /// </summary> + /// <param name="filePath">The path to the file.</param> + /// <param name="keyframeData">The keyframes.</param> + /// <returns>A value indicating whether the keyframe extraction was successful.</returns> + bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData); +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs new file mode 100644 index 000000000..4bc537c0e --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Extractors/MatroskaKeyframeExtractor.cs @@ -0,0 +1,48 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Jellyfin.MediaEncoding.Keyframes; +using Microsoft.Extensions.Logging; +using Extractor = Jellyfin.MediaEncoding.Keyframes.Matroska.MatroskaKeyframeExtractor; + +namespace Jellyfin.MediaEncoding.Hls.Extractors; + +/// <inheritdoc /> +public class MatroskaKeyframeExtractor : IKeyframeExtractor +{ + private readonly ILogger<MatroskaKeyframeExtractor> _logger; + + /// <summary> + /// Initializes a new instance of the <see cref="MatroskaKeyframeExtractor"/> class. + /// </summary> + /// <param name="logger">An instance of the <see cref="ILogger{MatroskaKeyframeExtractor}"/> interface.</param> + public MatroskaKeyframeExtractor(ILogger<MatroskaKeyframeExtractor> logger) + { + _logger = logger; + } + + /// <inheritdoc /> + public bool IsMetadataBased => true; + + /// <inheritdoc /> + public bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + if (!filePath.AsSpan().EndsWith(".mkv", StringComparison.OrdinalIgnoreCase)) + { + keyframeData = null; + return false; + } + + try + { + keyframeData = Extractor.GetKeyframeData(filePath); + return keyframeData.KeyframeTicks.Count > 0; + } + catch (Exception ex) + { + _logger.LogError(ex, "Extracting keyframes from {FilePath} using matroska metadata failed", filePath); + } + + keyframeData = null; + return false; + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj b/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj new file mode 100644 index 000000000..3f4f55ee4 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Jellyfin.MediaEncoding.Hls.csproj @@ -0,0 +1,35 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net7.0</TargetFramework> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <!-- Code Analyzers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + <PackageReference Include="SerilogAnalyzer" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" PrivateAssets="All" /> + </ItemGroup> + + <ItemGroup> + <ProjectReference Include="../../MediaBrowser.Common/MediaBrowser.Common.csproj" /> + <ProjectReference Include="../../MediaBrowser.Controller/MediaBrowser.Controller.csproj" /> + <ProjectReference Include="../Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" /> + </ItemGroup> + + <ItemGroup> + <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> + <_Parameter1>Jellyfin.MediaEncoding.Hls.Tests</_Parameter1> + </AssemblyAttribute> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs new file mode 100644 index 000000000..8572a5eae --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/CreateMainPlaylistRequest.cs @@ -0,0 +1,63 @@ +namespace Jellyfin.MediaEncoding.Hls.Playlist; + +/// <summary> +/// Request class for the <see cref="IDynamicHlsPlaylistGenerator.CreateMainPlaylist(CreateMainPlaylistRequest)"/> method. +/// </summary> +public class CreateMainPlaylistRequest +{ + /// <summary> + /// Initializes a new instance of the <see cref="CreateMainPlaylistRequest"/> class. + /// </summary> + /// <param name="filePath">The absolute file path to the file.</param> + /// <param name="desiredSegmentLengthMs">The desired segment length in milliseconds.</param> + /// <param name="totalRuntimeTicks">The total duration of the file in ticks.</param> + /// <param name="segmentContainer">The desired segment container eg. "ts".</param> + /// <param name="endpointPrefix">The URI prefix for the relative URL in the playlist.</param> + /// <param name="queryString">The desired query string to append (must start with ?).</param> + /// <param name="isRemuxingVideo">Whether the video is being remuxed.</param> + public CreateMainPlaylistRequest(string filePath, int desiredSegmentLengthMs, long totalRuntimeTicks, string segmentContainer, string endpointPrefix, string queryString, bool isRemuxingVideo) + { + FilePath = filePath; + DesiredSegmentLengthMs = desiredSegmentLengthMs; + TotalRuntimeTicks = totalRuntimeTicks; + SegmentContainer = segmentContainer; + EndpointPrefix = endpointPrefix; + QueryString = queryString; + IsRemuxingVideo = isRemuxingVideo; + } + + /// <summary> + /// Gets the file path. + /// </summary> + public string FilePath { get; } + + /// <summary> + /// Gets the desired segment length in milliseconds. + /// </summary> + public int DesiredSegmentLengthMs { get; } + + /// <summary> + /// Gets the total runtime in ticks. + /// </summary> + public long TotalRuntimeTicks { get; } + + /// <summary> + /// Gets the segment container. + /// </summary> + public string SegmentContainer { get; } + + /// <summary> + /// Gets the endpoint prefix for the URL. + /// </summary> + public string EndpointPrefix { get; } + + /// <summary> + /// Gets the query string. + /// </summary> + public string QueryString { get; } + + /// <summary> + /// Gets a value indicating whether the video is being remuxed. + /// </summary> + public bool IsRemuxingVideo { get; } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs new file mode 100644 index 000000000..07a6d6050 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/DynamicHlsPlaylistGenerator.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using Jellyfin.MediaEncoding.Hls.Extractors; +using Jellyfin.MediaEncoding.Keyframes; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.MediaEncoding; + +namespace Jellyfin.MediaEncoding.Hls.Playlist; + +/// <inheritdoc /> +public class DynamicHlsPlaylistGenerator : IDynamicHlsPlaylistGenerator +{ + private readonly IServerConfigurationManager _serverConfigurationManager; + private readonly IKeyframeExtractor[] _extractors; + + /// <summary> + /// Initializes a new instance of the <see cref="DynamicHlsPlaylistGenerator"/> class. + /// </summary> + /// <param name="serverConfigurationManager">An instance of the see <see cref="IServerConfigurationManager"/> interface.</param> + /// <param name="extractors">An instance of <see cref="IEnumerable{IKeyframeExtractor}"/>.</param> + public DynamicHlsPlaylistGenerator(IServerConfigurationManager serverConfigurationManager, IEnumerable<IKeyframeExtractor> extractors) + { + _serverConfigurationManager = serverConfigurationManager; + _extractors = extractors.Where(e => e.IsMetadataBased).ToArray(); + } + + /// <inheritdoc /> + public string CreateMainPlaylist(CreateMainPlaylistRequest request) + { + IReadOnlyList<double> segments; + // For video transcodes it is sufficient with equal length segments as ffmpeg will create new keyframes + if (request.IsRemuxingVideo && TryExtractKeyframes(request.FilePath, out var keyframeData)) + { + segments = ComputeSegments(keyframeData, request.DesiredSegmentLengthMs); + } + else + { + segments = ComputeEqualLengthSegments(request.DesiredSegmentLengthMs, request.TotalRuntimeTicks); + } + + var segmentExtension = EncodingHelper.GetSegmentFileExtension(request.SegmentContainer); + + // http://ffmpeg.org/ffmpeg-all.html#toc-hls-2 + var isHlsInFmp4 = string.Equals(segmentExtension, ".mp4", StringComparison.OrdinalIgnoreCase); + var hlsVersion = isHlsInFmp4 ? "7" : "3"; + + var builder = new StringBuilder(128); + + builder.AppendLine("#EXTM3U") + .AppendLine("#EXT-X-PLAYLIST-TYPE:VOD") + .Append("#EXT-X-VERSION:") + .Append(hlsVersion) + .AppendLine() + .Append("#EXT-X-TARGETDURATION:") + .Append(Math.Ceiling(segments.Count > 0 ? segments.Max() : request.DesiredSegmentLengthMs)) + .AppendLine() + .AppendLine("#EXT-X-MEDIA-SEQUENCE:0"); + + var index = 0; + + if (isHlsInFmp4) + { + // Init file that only includes fMP4 headers + builder.Append("#EXT-X-MAP:URI=\"") + .Append(request.EndpointPrefix) + .Append("-1") + .Append(segmentExtension) + .Append(request.QueryString) + .Append("&runtimeTicks=0") + .Append("&actualSegmentLengthTicks=0") + .Append('"') + .AppendLine(); + } + + long currentRuntimeInSeconds = 0; + foreach (var length in segments) + { + // Manually convert to ticks to avoid precision loss when converting double + var lengthTicks = Convert.ToInt64(length * TimeSpan.TicksPerSecond); + builder.Append("#EXTINF:") + .Append(length.ToString("0.000000", CultureInfo.InvariantCulture)) + .AppendLine(", nodesc") + .Append(request.EndpointPrefix) + .Append(index++) + .Append(segmentExtension) + .Append(request.QueryString) + .Append("&runtimeTicks=") + .Append(currentRuntimeInSeconds) + .Append("&actualSegmentLengthTicks=") + .Append(lengthTicks) + .AppendLine(); + + currentRuntimeInSeconds += lengthTicks; + } + + builder.AppendLine("#EXT-X-ENDLIST"); + + return builder.ToString(); + } + + private bool TryExtractKeyframes(string filePath, [NotNullWhen(true)] out KeyframeData? keyframeData) + { + keyframeData = null; + if (!IsExtractionAllowedForFile(filePath, _serverConfigurationManager.GetEncodingOptions().AllowOnDemandMetadataBasedKeyframeExtractionForExtensions)) + { + return false; + } + + var len = _extractors.Length; + for (var i = 0; i < len; i++) + { + var extractor = _extractors[i]; + if (!extractor.TryExtractKeyframes(filePath, out var result)) + { + continue; + } + + keyframeData = result; + return true; + } + + return false; + } + + internal static bool IsExtractionAllowedForFile(ReadOnlySpan<char> filePath, string[] allowedExtensions) + { + var extension = Path.GetExtension(filePath); + if (extension.IsEmpty) + { + return false; + } + + // Remove the leading dot + var extensionWithoutDot = extension[1..]; + for (var i = 0; i < allowedExtensions.Length; i++) + { + var allowedExtension = allowedExtensions[i].AsSpan().TrimStart('.'); + if (extensionWithoutDot.Equals(allowedExtension, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + internal static IReadOnlyList<double> ComputeSegments(KeyframeData keyframeData, int desiredSegmentLengthMs) + { + if (keyframeData.KeyframeTicks.Count > 0 && keyframeData.TotalDuration < keyframeData.KeyframeTicks[^1]) + { + throw new ArgumentException("Invalid duration in keyframe data", nameof(keyframeData)); + } + + long lastKeyframe = 0; + var result = new List<double>(); + // Scale the segment length to ticks to match the keyframes + var desiredSegmentLengthTicks = TimeSpan.FromMilliseconds(desiredSegmentLengthMs).Ticks; + var desiredCutTime = desiredSegmentLengthTicks; + for (var j = 0; j < keyframeData.KeyframeTicks.Count; j++) + { + var keyframe = keyframeData.KeyframeTicks[j]; + if (keyframe >= desiredCutTime) + { + var currentSegmentLength = keyframe - lastKeyframe; + result.Add(TimeSpan.FromTicks(currentSegmentLength).TotalSeconds); + lastKeyframe = keyframe; + desiredCutTime += desiredSegmentLengthTicks; + } + } + + result.Add(TimeSpan.FromTicks(keyframeData.TotalDuration - lastKeyframe).TotalSeconds); + return result; + } + + internal static double[] ComputeEqualLengthSegments(int desiredSegmentLengthMs, long totalRuntimeTicks) + { + if (desiredSegmentLengthMs == 0 || totalRuntimeTicks == 0) + { + throw new InvalidOperationException($"Invalid segment length ({desiredSegmentLengthMs}) or runtime ticks ({totalRuntimeTicks})"); + } + + var desiredSegmentLength = TimeSpan.FromMilliseconds(desiredSegmentLengthMs); + + var segmentLengthTicks = desiredSegmentLength.Ticks; + var wholeSegments = totalRuntimeTicks / segmentLengthTicks; + var remainingTicks = totalRuntimeTicks % segmentLengthTicks; + + var segmentsLen = wholeSegments + (remainingTicks == 0 ? 0 : 1); + var segments = new double[segmentsLen]; + for (int i = 0; i < wholeSegments; i++) + { + segments[i] = desiredSegmentLength.TotalSeconds; + } + + if (remainingTicks != 0) + { + segments[^1] = TimeSpan.FromTicks(remainingTicks).TotalSeconds; + } + + return segments; + } +} diff --git a/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs b/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs new file mode 100644 index 000000000..2626cb2dd --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/Playlist/IDynamicHlsPlaylistGenerator.cs @@ -0,0 +1,14 @@ +namespace Jellyfin.MediaEncoding.Hls.Playlist; + +/// <summary> +/// Generator for dynamic HLS playlists where the segment lengths aren't known in advance. +/// </summary> +public interface IDynamicHlsPlaylistGenerator +{ + /// <summary> + /// Creates the main playlist containing the main video or audio stream. + /// </summary> + /// <param name="request">An instance of the <see cref="CreateMainPlaylistRequest"/> class.</param> + /// <returns>The playlist as a formatted string.</returns> + string CreateMainPlaylist(CreateMainPlaylistRequest request); +} diff --git a/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs new file mode 100644 index 000000000..a7d52184b --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Hls/ScheduledTasks/KeyframeExtractionScheduledTask.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Enums; +using Jellyfin.MediaEncoding.Hls.Extractors; +using MediaBrowser.Controller.Dto; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Globalization; +using MediaBrowser.Model.Tasks; + +namespace Jellyfin.MediaEncoding.Hls.ScheduledTasks; + +/// <inheritdoc /> +public class KeyframeExtractionScheduledTask : IScheduledTask +{ + private const int Pagesize = 1000; + + private readonly ILocalizationManager _localizationManager; + private readonly ILibraryManager _libraryManager; + private readonly IKeyframeExtractor[] _keyframeExtractors; + private static readonly BaseItemKind[] _itemTypes = { BaseItemKind.Episode, BaseItemKind.Movie }; + + /// <summary> + /// Initializes a new instance of the <see cref="KeyframeExtractionScheduledTask"/> class. + /// </summary> + /// <param name="localizationManager">An instance of the <see cref="ILocalizationManager"/> interface.</param> + /// <param name="libraryManager">An instance of the <see cref="ILibraryManager"/> interface.</param> + /// <param name="keyframeExtractors">The keyframe extractors.</param> + public KeyframeExtractionScheduledTask(ILocalizationManager localizationManager, ILibraryManager libraryManager, IEnumerable<IKeyframeExtractor> keyframeExtractors) + { + _localizationManager = localizationManager; + _libraryManager = libraryManager; + _keyframeExtractors = keyframeExtractors.OrderByDescending(e => e.IsMetadataBased).ToArray(); + } + + /// <inheritdoc /> + public string Name => _localizationManager.GetLocalizedString("TaskKeyframeExtractor"); + + /// <inheritdoc /> + public string Key => "KeyframeExtraction"; + + /// <inheritdoc /> + public string Description => _localizationManager.GetLocalizedString("TaskKeyframeExtractorDescription"); + + /// <inheritdoc /> + public string Category => _localizationManager.GetLocalizedString("TasksLibraryCategory"); + + /// <inheritdoc /> + public Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken) + { + var query = new InternalItemsQuery + { + MediaTypes = new[] { MediaType.Video }, + IsVirtualItem = false, + IncludeItemTypes = _itemTypes, + DtoOptions = new DtoOptions(true), + SourceTypes = new[] { SourceType.Library }, + Recursive = true, + Limit = Pagesize + }; + + var numberOfVideos = _libraryManager.GetCount(query); + + var startIndex = 0; + var numComplete = 0; + + while (startIndex < numberOfVideos) + { + query.StartIndex = startIndex; + + var videos = _libraryManager.GetItemList(query); + var currentPageCount = videos.Count; + // TODO parallelize with Parallel.ForEach? + for (var i = 0; i < currentPageCount; i++) + { + var video = videos[i]; + // Only local files supported + if (video.IsFileProtocol && File.Exists(video.Path)) + { + for (var j = 0; j < _keyframeExtractors.Length; j++) + { + var extractor = _keyframeExtractors[j]; + // The cache decorator will make sure to save them in the data dir + if (extractor.TryExtractKeyframes(video.Path, out _)) + { + break; + } + } + } + + // Update progress + numComplete++; + double percent = (double)numComplete / numberOfVideos; + progress.Report(100 * percent); + } + + startIndex += Pagesize; + } + + progress.Report(100); + return Task.CompletedTask; + } + + /// <inheritdoc /> + public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() => Enumerable.Empty<TaskTriggerInfo>(); +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs new file mode 100644 index 000000000..479e6ffdc --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfProbe/FfProbeKeyframeExtractor.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace Jellyfin.MediaEncoding.Keyframes.FfProbe; + +/// <summary> +/// FfProbe based keyframe extractor. +/// </summary> +public static class FfProbeKeyframeExtractor +{ + private const string DefaultArguments = "-fflags +genpts -v error -skip_frame nokey -show_entries format=duration -show_entries stream=duration -show_entries packet=pts_time,flags -select_streams v -of csv \"{0}\""; + + /// <summary> + /// Extracts the keyframes using the ffprobe executable at the specified path. + /// </summary> + /// <param name="ffProbePath">The path to the ffprobe executable.</param> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string ffProbePath, string filePath) + { + using var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = ffProbePath, + Arguments = string.Format(CultureInfo.InvariantCulture, DefaultArguments, filePath), + + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + + WindowStyle = ProcessWindowStyle.Hidden, + ErrorDialog = false, + }, + EnableRaisingEvents = true + }; + + try + { + process.Start(); + + return ParseStream(process.StandardOutput); + } + catch (Exception) + { + try + { + if (!process.HasExited) + { + process.Kill(); + } + } + catch + { + // We do not care if this fails + } + + throw; + } + } + + internal static KeyframeData ParseStream(StreamReader reader) + { + var keyframes = new List<long>(); + double streamDuration = 0; + double formatDuration = 0; + + using (reader) + { + while (!reader.EndOfStream) + { + var line = reader.ReadLine().AsSpan(); + if (line.IsEmpty) + { + continue; + } + + var firstComma = line.IndexOf(','); + var lineType = line[..firstComma]; + var rest = line[(firstComma + 1)..]; + if (lineType.Equals("packet", StringComparison.OrdinalIgnoreCase)) + { + // Split time and flags from the packet line. Example line: packet,7169.079000,K_ + var secondComma = rest.IndexOf(','); + var ptsTime = rest[..secondComma]; + var flags = rest[(secondComma + 1)..]; + if (flags.StartsWith("K_")) + { + if (double.TryParse(ptsTime, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var keyframe)) + { + // Have to manually convert to ticks to avoid rounding errors as TimeSpan is only precise down to 1 ms when converting double. + keyframes.Add(Convert.ToInt64(keyframe * TimeSpan.TicksPerSecond)); + } + } + } + else if (lineType.Equals("stream", StringComparison.OrdinalIgnoreCase)) + { + if (double.TryParse(rest, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var streamDurationResult)) + { + streamDuration = streamDurationResult; + } + } + else if (lineType.Equals("format", StringComparison.OrdinalIgnoreCase)) + { + if (double.TryParse(rest, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var formatDurationResult)) + { + formatDuration = formatDurationResult; + } + } + } + + // Prefer the stream duration as it should be more accurate + var duration = streamDuration > 0 ? streamDuration : formatDuration; + + return new KeyframeData(TimeSpan.FromSeconds(duration).Ticks, keyframes); + } + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs new file mode 100644 index 000000000..aaaca6fe1 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/FfTool/FfToolKeyframeExtractor.cs @@ -0,0 +1,17 @@ +using System; + +namespace Jellyfin.MediaEncoding.Keyframes.FfTool; + +/// <summary> +/// FfTool based keyframe extractor. +/// </summary> +public static class FfToolKeyframeExtractor +{ + /// <summary> + /// Extracts the keyframes using the fftool executable at the specified path. + /// </summary> + /// <param name="ffToolPath">The path to the fftool executable.</param> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string ffToolPath, string filePath) => throw new NotImplementedException(); +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj new file mode 100644 index 000000000..71572bcf6 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Jellyfin.MediaEncoding.Keyframes.csproj @@ -0,0 +1,33 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net7.0</TargetFramework> + <GenerateDocumentationFile>true</GenerateDocumentationFile> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="NEbml" /> + </ItemGroup> + + <!-- Code Analyzers--> + <ItemGroup Condition=" '$(Configuration)' == 'Debug' "> + <PackageReference Include="Microsoft.CodeAnalysis.BannedApiAnalyzers"> + <PrivateAssets>all</PrivateAssets> + <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> + </PackageReference> + <PackageReference Include="SerilogAnalyzer" PrivateAssets="All" /> + <PackageReference Include="StyleCop.Analyzers" PrivateAssets="All" /> + <PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer" PrivateAssets="All" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" /> + </ItemGroup> + + <ItemGroup> + <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo"> + <_Parameter1>Jellyfin.MediaEncoding.Keyframes.Tests</_Parameter1> + </AssemblyAttribute> + </ItemGroup> + +</Project> diff --git a/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs new file mode 100644 index 000000000..06f9180e7 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/KeyframeData.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; + +namespace Jellyfin.MediaEncoding.Keyframes; + +/// <summary> +/// Keyframe information for a specific file. +/// </summary> +public class KeyframeData +{ + /// <summary> + /// Initializes a new instance of the <see cref="KeyframeData"/> class. + /// </summary> + /// <param name="totalDuration">The total duration of the video stream in ticks.</param> + /// <param name="keyframeTicks">The video keyframes in ticks.</param> + public KeyframeData(long totalDuration, IReadOnlyList<long> keyframeTicks) + { + TotalDuration = totalDuration; + KeyframeTicks = keyframeTicks; + } + + /// <summary> + /// Gets the total duration of the stream in ticks. + /// </summary> + public long TotalDuration { get; } + + /// <summary> + /// Gets the keyframes in ticks. + /// </summary> + public IReadOnlyList<long> KeyframeTicks { get; } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs new file mode 100644 index 000000000..fd170864b --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Extensions/EbmlReaderExtensions.cs @@ -0,0 +1,177 @@ +using System; +using System.Buffers.Binary; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Models; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions; + +/// <summary> +/// Extension methods for the <see cref="EbmlReader"/> class. +/// </summary> +internal static class EbmlReaderExtensions +{ + /// <summary> + /// Traverses the current container to find the element with <paramref name="identifier"/> identifier. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <param name="identifier">The element identifier.</param> + /// <returns>A value indicating whether the element was found.</returns> + internal static bool FindElement(this EbmlReader reader, ulong identifier) + { + while (reader.ReadNext()) + { + if (reader.ElementId.EncodedValue == identifier) + { + return true; + } + } + + return false; + } + + /// <summary> + /// Reads the current position in the file as an unsigned integer converted from binary. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <returns>The unsigned integer.</returns> + internal static uint ReadUIntFromBinary(this EbmlReader reader) + { + var buffer = new byte[4]; + reader.ReadBinary(buffer, 0, 4); + return BinaryPrimitives.ReadUInt32BigEndian(buffer); + } + + /// <summary> + /// Reads from the start of the file to retrieve the SeekHead segment. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <returns>Instance of <see cref="SeekHead"/>.</returns> + internal static SeekHead ReadSeekHead(this EbmlReader reader) + { + reader = reader ?? throw new ArgumentNullException(nameof(reader)); + + if (reader.ElementPosition != 0) + { + throw new InvalidOperationException("File position must be at 0"); + } + + // Skip the header + if (!reader.FindElement(MatroskaConstants.SegmentContainer)) + { + throw new InvalidOperationException("Expected a segment container"); + } + + reader.EnterContainer(); + + long? tracksPosition = null; + long? cuesPosition = null; + long? infoPosition = null; + // The first element should be a SeekHead otherwise we'll have to search manually + if (!reader.FindElement(MatroskaConstants.SeekHead)) + { + throw new InvalidOperationException("Expected a SeekHead"); + } + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.Seek)) + { + reader.EnterContainer(); + reader.ReadNext(); + var type = (ulong)reader.ReadUIntFromBinary(); + switch (type) + { + case MatroskaConstants.Tracks: + reader.ReadNext(); + tracksPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Cues: + reader.ReadNext(); + cuesPosition = (long)reader.ReadUInt(); + break; + case MatroskaConstants.Info: + reader.ReadNext(); + infoPosition = (long)reader.ReadUInt(); + break; + } + + reader.LeaveContainer(); + + if (tracksPosition.HasValue && cuesPosition.HasValue && infoPosition.HasValue) + { + break; + } + } + + reader.LeaveContainer(); + + if (!tracksPosition.HasValue || !cuesPosition.HasValue || !infoPosition.HasValue) + { + throw new InvalidOperationException("SeekHead is missing or does not contain Info, Tracks and Cues positions. SeekHead referencing another SeekHead is not supported"); + } + + return new SeekHead(infoPosition.Value, tracksPosition.Value, cuesPosition.Value); + } + + /// <summary> + /// Reads from SegmentContainer to retrieve the Info segment. + /// </summary> + /// <param name="reader">An instance of <see cref="EbmlReader"/>.</param> + /// <param name="position">The position of the info segment relative to the Segment container.</param> + /// <returns>Instance of <see cref="Info"/>.</returns> + internal static Info ReadInfo(this EbmlReader reader, long position) + { + reader.ReadAt(position); + + double? duration = null; + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TimestampScale); + var timestampScale = reader.ReadUInt(); + + if (reader.FindElement(MatroskaConstants.Duration)) + { + duration = reader.ReadFloat(); + } + + reader.LeaveContainer(); + + return new Info((long)timestampScale, duration); + } + + /// <summary> + /// Enters the Tracks segment and reads all tracks to find the specified type. + /// </summary> + /// <param name="reader">Instance of <see cref="EbmlReader"/>.</param> + /// <param name="tracksPosition">The relative position of the tracks segment.</param> + /// <param name="type">The track type identifier.</param> + /// <returns>The first track number with the specified type.</returns> + /// <exception cref="InvalidOperationException">Stream type is not found.</exception> + internal static ulong FindFirstTrackNumberByType(this EbmlReader reader, long tracksPosition, ulong type) + { + reader.ReadAt(tracksPosition); + + reader.EnterContainer(); + while (reader.FindElement(MatroskaConstants.TrackEntry)) + { + reader.EnterContainer(); + // Mandatory element + reader.FindElement(MatroskaConstants.TrackNumber); + var trackNumber = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.TrackType); + var trackType = reader.ReadUInt(); + + reader.LeaveContainer(); + if (trackType == MatroskaConstants.TrackTypeVideo) + { + reader.LeaveContainer(); + return trackNumber; + } + } + + reader.LeaveContainer(); + + throw new InvalidOperationException($"No stream with type {type} found"); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs new file mode 100644 index 000000000..0d5c2f34f --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaConstants.cs @@ -0,0 +1,30 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska; + +/// <summary> +/// Constants for the Matroska identifiers. +/// </summary> +public static class MatroskaConstants +{ + internal const ulong SegmentContainer = 0x18538067; + + internal const ulong SeekHead = 0x114D9B74; + internal const ulong Seek = 0x4DBB; + + internal const ulong Info = 0x1549A966; + internal const ulong TimestampScale = 0x2AD7B1; + internal const ulong Duration = 0x4489; + + internal const ulong Tracks = 0x1654AE6B; + internal const ulong TrackEntry = 0xAE; + internal const ulong TrackNumber = 0xD7; + internal const ulong TrackType = 0x83; + + internal const ulong TrackTypeVideo = 0x1; + internal const ulong TrackTypeSubtitle = 0x11; + + internal const ulong Cues = 0x1C53BB6B; + internal const ulong CueTime = 0xB3; + internal const ulong CuePoint = 0xBB; + internal const ulong CueTrackPositions = 0xB7; + internal const ulong CuePointTrackNumber = 0xF7; +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs new file mode 100644 index 000000000..501b2bb17 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/MatroskaKeyframeExtractor.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Extensions; +using Jellyfin.MediaEncoding.Keyframes.Matroska.Models; +using NEbml.Core; + +namespace Jellyfin.MediaEncoding.Keyframes.Matroska; + +/// <summary> +/// The keyframe extractor for the matroska container. +/// </summary> +public static class MatroskaKeyframeExtractor +{ + /// <summary> + /// Extracts the keyframes in ticks (scaled using the container timestamp scale) from the matroska container. + /// </summary> + /// <param name="filePath">The file path.</param> + /// <returns>An instance of <see cref="KeyframeData"/>.</returns> + public static KeyframeData GetKeyframeData(string filePath) + { + using var stream = File.OpenRead(filePath); + using var reader = new EbmlReader(stream); + + var seekHead = reader.ReadSeekHead(); + // External lib does not support seeking backwards (yet) + Info info; + ulong videoTrackNumber; + if (seekHead.InfoPosition < seekHead.TracksPosition) + { + info = reader.ReadInfo(seekHead.InfoPosition); + videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + } + else + { + videoTrackNumber = reader.FindFirstTrackNumberByType(seekHead.TracksPosition, MatroskaConstants.TrackTypeVideo); + info = reader.ReadInfo(seekHead.InfoPosition); + } + + var keyframes = new List<long>(); + reader.ReadAt(seekHead.CuesPosition); + reader.EnterContainer(); + + while (reader.FindElement(MatroskaConstants.CuePoint)) + { + reader.EnterContainer(); + ulong? trackNumber = null; + // Mandatory element + reader.FindElement(MatroskaConstants.CueTime); + var cueTime = reader.ReadUInt(); + + // Mandatory element + reader.FindElement(MatroskaConstants.CueTrackPositions); + reader.EnterContainer(); + if (reader.FindElement(MatroskaConstants.CuePointTrackNumber)) + { + trackNumber = reader.ReadUInt(); + } + + reader.LeaveContainer(); + + if (trackNumber == videoTrackNumber) + { + keyframes.Add(ScaleToTicks(cueTime, info.TimestampScale)); + } + + reader.LeaveContainer(); + } + + reader.LeaveContainer(); + + var result = new KeyframeData(ScaleToTicks(info.Duration ?? 0, info.TimestampScale), keyframes); + return result; + } + + private static long ScaleToTicks(ulong unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return (long)unscaledValue * timestampScale / 100; + } + + private static long ScaleToTicks(double unscaledValue, long timestampScale) + { + // TimestampScale is in nanoseconds, scale it to get the value in ticks, 1 tick == 100 ns + return Convert.ToInt64(unscaledValue * timestampScale / 100); + } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs new file mode 100644 index 000000000..415d6da00 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/Info.cs @@ -0,0 +1,28 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models; + +/// <summary> +/// The matroska Info segment. +/// </summary> +internal class Info +{ + /// <summary> + /// Initializes a new instance of the <see cref="Info"/> class. + /// </summary> + /// <param name="timestampScale">The timestamp scale in nanoseconds.</param> + /// <param name="duration">The duration of the entire file.</param> + public Info(long timestampScale, double? duration) + { + TimestampScale = timestampScale; + Duration = duration; + } + + /// <summary> + /// Gets the timestamp scale in nanoseconds. + /// </summary> + public long TimestampScale { get; } + + /// <summary> + /// Gets the total duration of the file. + /// </summary> + public double? Duration { get; } +} diff --git a/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs new file mode 100644 index 000000000..95e4fd882 --- /dev/null +++ b/src/Jellyfin.MediaEncoding.Keyframes/Matroska/Models/SeekHead.cs @@ -0,0 +1,35 @@ +namespace Jellyfin.MediaEncoding.Keyframes.Matroska.Models; + +/// <summary> +/// The matroska SeekHead segment. All positions are relative to the Segment container. +/// </summary> +internal class SeekHead +{ + /// <summary> + /// Initializes a new instance of the <see cref="SeekHead"/> class. + /// </summary> + /// <param name="infoPosition">The relative file position of the info segment.</param> + /// <param name="tracksPosition">The relative file position of the tracks segment.</param> + /// <param name="cuesPosition">The relative file position of the cues segment.</param> + public SeekHead(long infoPosition, long tracksPosition, long cuesPosition) + { + InfoPosition = infoPosition; + TracksPosition = tracksPosition; + CuesPosition = cuesPosition; + } + + /// <summary> + /// Gets relative file position of the info segment. + /// </summary> + public long InfoPosition { get; } + + /// <summary> + /// Gets the relative file position of the tracks segment. + /// </summary> + public long TracksPosition { get; } + + /// <summary> + /// Gets the relative file position of the cues segment. + /// </summary> + public long CuesPosition { get; } +} |