diff options
32 files changed, 2278 insertions, 7 deletions
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index e3af12a49..dc5f99c0c 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -57,6 +57,7 @@ - [hawken93](https://github.com/hawken93) - [HelloWorld017](https://github.com/HelloWorld017) - [ikomhoog](https://github.com/ikomhoog) + - [iwalton3](https://github.com/iwalton3) - [jftuga](https://github.com/jftuga) - [jmshrv](https://github.com/jmshrv) - [joern-h](https://github.com/joern-h) @@ -88,6 +89,7 @@ - [neilsb](https://github.com/neilsb) - [nevado](https://github.com/nevado) - [Nickbert7](https://github.com/Nickbert7) + - [nicknsy](https://github.com/nicknsy) - [nvllsvm](https://github.com/nvllsvm) - [nyanmisaka](https://github.com/nyanmisaka) - [OancaAndrei](https://github.com/OancaAndrei) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 6d27703bd..44b97e8b8 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -22,6 +22,7 @@ using MediaBrowser.Controller.Lyrics; using MediaBrowser.Controller.Persistence; using MediaBrowser.Controller.Playlists; using MediaBrowser.Controller.Providers; +using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Querying; @@ -52,6 +53,7 @@ namespace Emby.Server.Implementations.Dto private readonly Lazy<ILiveTvManager> _livetvManagerFactory; private readonly ILyricManager _lyricManager; + private readonly ITrickplayManager _trickplayManager; public DtoService( ILogger<DtoService> logger, @@ -63,7 +65,8 @@ namespace Emby.Server.Implementations.Dto IApplicationHost appHost, IMediaSourceManager mediaSourceManager, Lazy<ILiveTvManager> livetvManagerFactory, - ILyricManager lyricManager) + ILyricManager lyricManager, + ITrickplayManager trickplayManager) { _logger = logger; _libraryManager = libraryManager; @@ -75,6 +78,7 @@ namespace Emby.Server.Implementations.Dto _mediaSourceManager = mediaSourceManager; _livetvManagerFactory = livetvManagerFactory; _lyricManager = lyricManager; + _trickplayManager = trickplayManager; } private ILiveTvManager LivetvManager => _livetvManagerFactory.Value; @@ -1059,6 +1063,11 @@ namespace Emby.Server.Implementations.Dto dto.Chapters = _itemRepo.GetChapters(item); } + if (options.ContainsField(ItemFields.Trickplay)) + { + dto.Trickplay = _trickplayManager.GetTrickplayManifest(item).GetAwaiter().GetResult(); + } + if (video.ExtraType.HasValue) { dto.ExtraType = video.ExtraType.Value.ToString(); diff --git a/Emby.Server.Implementations/Localization/Core/en-US.json b/Emby.Server.Implementations/Localization/Core/en-US.json index 15088384c..496ecabd3 100644 --- a/Emby.Server.Implementations/Localization/Core/en-US.json +++ b/Emby.Server.Implementations/Localization/Core/en-US.json @@ -112,6 +112,8 @@ "TaskCleanLogsDescription": "Deletes log files that are more than {0} days old.", "TaskRefreshPeople": "Refresh People", "TaskRefreshPeopleDescription": "Updates metadata for actors and directors in your media library.", + "TaskRefreshTrickplayImages": "Generate Trickplay Images", + "TaskRefreshTrickplayImagesDescription": "Creates trickplay previews for videos in enabled libraries.", "TaskUpdatePlugins": "Update Plugins", "TaskUpdatePluginsDescription": "Downloads and installs updates for plugins that are configured to update automatically.", "TaskCleanTranscode": "Clean Transcode Directory", diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index 42c94c29d..38953dc21 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -410,6 +410,7 @@ public class DynamicHlsController : BaseJellyfinApiController /// <param name="context">Optional. The <see cref="EncodingContext"/>.</param> /// <param name="streamOptions">Optional. The streaming options.</param> /// <param name="enableAdaptiveBitrateStreaming">Enable adaptive bitrate streaming.</param> + /// <param name="enableTrickplay">Enable trickplay image playlists being added to master playlist.</param> /// <response code="200">Video stream returned.</response> /// <returns>A <see cref="FileResult"/> containing the playlist file.</returns> [HttpGet("Videos/{itemId}/master.m3u8")] @@ -467,7 +468,8 @@ public class DynamicHlsController : BaseJellyfinApiController [FromQuery] int? videoStreamIndex, [FromQuery] EncodingContext? context, [FromQuery] Dictionary<string, string> streamOptions, - [FromQuery] bool enableAdaptiveBitrateStreaming = true) + [FromQuery] bool enableAdaptiveBitrateStreaming = true, + [FromQuery] bool enableTrickplay = true) { var streamingRequest = new HlsVideoRequestDto { @@ -521,7 +523,8 @@ public class DynamicHlsController : BaseJellyfinApiController VideoStreamIndex = videoStreamIndex, Context = context ?? EncodingContext.Streaming, StreamOptions = streamOptions, - EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming + EnableAdaptiveBitrateStreaming = enableAdaptiveBitrateStreaming, + EnableTrickplay = enableTrickplay }; return await _dynamicHlsHelper.GetMasterHlsPlaylist(TranscodingJobType, streamingRequest, enableAdaptiveBitrateStreaming).ConfigureAwait(false); diff --git a/Jellyfin.Api/Controllers/TrickplayController.cs b/Jellyfin.Api/Controllers/TrickplayController.cs new file mode 100644 index 000000000..2dc960229 --- /dev/null +++ b/Jellyfin.Api/Controllers/TrickplayController.cs @@ -0,0 +1,101 @@ +using System; +using System.ComponentModel.DataAnnotations; +using System.Net.Mime; +using System.Text; +using System.Threading.Tasks; +using Jellyfin.Api.Attributes; +using Jellyfin.Api.Extensions; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Trickplay; +using MediaBrowser.Model; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace Jellyfin.Api.Controllers; + +/// <summary> +/// Trickplay controller. +/// </summary> +[Route("")] +[Authorize] +public class TrickplayController : BaseJellyfinApiController +{ + private readonly ILibraryManager _libraryManager; + private readonly ITrickplayManager _trickplayManager; + + /// <summary> + /// Initializes a new instance of the <see cref="TrickplayController"/> class. + /// </summary> + /// <param name="libraryManager">Instance of <see cref="ILibraryManager"/>.</param> + /// <param name="trickplayManager">Instance of <see cref="ITrickplayManager"/>.</param> + public TrickplayController( + ILibraryManager libraryManager, + ITrickplayManager trickplayManager) + { + _libraryManager = libraryManager; + _trickplayManager = trickplayManager; + } + + /// <summary> + /// Gets an image tiles playlist for trickplay. + /// </summary> + /// <param name="itemId">The item id.</param> + /// <param name="width">The width of a single tile.</param> + /// <param name="mediaSourceId">The media version id, if using an alternate version.</param> + /// <response code="200">Tiles playlist returned.</response> + /// <returns>A <see cref="FileResult"/> containing the trickplay playlist file.</returns> + [HttpGet("Videos/{itemId}/Trickplay/{width}/tiles.m3u8")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [ProducesPlaylistFile] + public async Task<ActionResult> GetTrickplayHlsPlaylist( + [FromRoute, Required] Guid itemId, + [FromRoute, Required] int width, + [FromQuery] Guid? mediaSourceId) + { + string? playlist = await _trickplayManager.GetHlsPlaylist(mediaSourceId ?? itemId, width, User.GetToken()).ConfigureAwait(false); + + if (string.IsNullOrEmpty(playlist)) + { + return NotFound(); + } + + return Content(playlist, MimeTypes.GetMimeType("playlist.m3u8"), Encoding.UTF8); + } + + /// <summary> + /// Gets a trickplay tile image. + /// </summary> + /// <param name="itemId">The item id.</param> + /// <param name="width">The width of a single tile.</param> + /// <param name="index">The index of the desired tile.</param> + /// <param name="mediaSourceId">The media version id, if using an alternate version.</param> + /// <response code="200">Tile image returned.</response> + /// <response code="200">Tile image not found at specified index.</response> + /// <returns>A <see cref="FileResult"/> containing the trickplay tiles image.</returns> + [HttpGet("Videos/{itemId}/Trickplay/{width}/{index}.jpg")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status404NotFound)] + [ProducesImageFile] + public ActionResult GetTrickplayTileImage( + [FromRoute, Required] Guid itemId, + [FromRoute, Required] int width, + [FromRoute, Required] int index, + [FromQuery] Guid? mediaSourceId) + { + var item = _libraryManager.GetItemById(mediaSourceId ?? itemId); + if (item is null) + { + return NotFound(); + } + + var path = _trickplayManager.GetTrickplayTilePath(item, width, index); + if (System.IO.File.Exists(path)) + { + return PhysicalFile(path, MediaTypeNames.Image.Jpeg); + } + + return NotFound(); + } +} diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index 276a09f41..24082fcff 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Jellyfin.Api.Extensions; using Jellyfin.Api.Models.StreamingDtos; +using Jellyfin.Data.Entities; using Jellyfin.Data.Enums; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; @@ -19,6 +20,7 @@ using MediaBrowser.Controller.Devices; using MediaBrowser.Controller.Dlna; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Net; @@ -46,6 +48,7 @@ public class DynamicHlsHelper private readonly ILogger<DynamicHlsHelper> _logger; private readonly IHttpContextAccessor _httpContextAccessor; private readonly EncodingHelper _encodingHelper; + private readonly ITrickplayManager _trickplayManager; /// <summary> /// Initializes a new instance of the <see cref="DynamicHlsHelper"/> class. @@ -62,6 +65,7 @@ public class DynamicHlsHelper /// <param name="logger">Instance of the <see cref="ILogger{DynamicHlsHelper}"/> interface.</param> /// <param name="httpContextAccessor">Instance of the <see cref="IHttpContextAccessor"/> interface.</param> /// <param name="encodingHelper">Instance of <see cref="EncodingHelper"/>.</param> + /// <param name="trickplayManager">Instance of <see cref="ITrickplayManager"/>.</param> public DynamicHlsHelper( ILibraryManager libraryManager, IUserManager userManager, @@ -74,7 +78,8 @@ public class DynamicHlsHelper INetworkManager networkManager, ILogger<DynamicHlsHelper> logger, IHttpContextAccessor httpContextAccessor, - EncodingHelper encodingHelper) + EncodingHelper encodingHelper, + ITrickplayManager trickplayManager) { _libraryManager = libraryManager; _userManager = userManager; @@ -88,6 +93,7 @@ public class DynamicHlsHelper _logger = logger; _httpContextAccessor = httpContextAccessor; _encodingHelper = encodingHelper; + _trickplayManager = trickplayManager; } /// <summary> @@ -280,6 +286,13 @@ public class DynamicHlsHelper AppendPlaylist(builder, state, variantUrl, newBitrate, subtitleGroup); } + if (!isLiveStream && (state.VideoRequest?.EnableTrickplay ?? false)) + { + var sourceId = Guid.Parse(state.Request.MediaSourceId); + var trickplayResolutions = await _trickplayManager.GetTrickplayResolutions(sourceId).ConfigureAwait(false); + AddTrickplay(state, trickplayResolutions, builder, _httpContextAccessor.HttpContext.User); + } + return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8")); } @@ -509,6 +522,41 @@ public class DynamicHlsHelper } /// <summary> + /// Appends EXT-X-IMAGE-STREAM-INF playlists for each available trickplay resolution. + /// </summary> + /// <param name="state">StreamState of the current stream.</param> + /// <param name="trickplayResolutions">Dictionary of widths to corresponding tiles info.</param> + /// <param name="builder">StringBuilder to append the field to.</param> + /// <param name="user">Http user context.</param> + private void AddTrickplay(StreamState state, Dictionary<int, TrickplayInfo> trickplayResolutions, StringBuilder builder, ClaimsPrincipal user) + { + const string playlistFormat = "#EXT-X-IMAGE-STREAM-INF:BANDWIDTH={0},RESOLUTION={1}x{2},CODECS=\"jpeg\",URI=\"{3}\""; + + foreach (var resolution in trickplayResolutions) + { + var width = resolution.Key; + var trickplayInfo = resolution.Value; + + var url = string.Format( + CultureInfo.InvariantCulture, + "Trickplay/{0}/tiles.m3u8?MediaSourceId={1}&api_key={2}", + width.ToString(CultureInfo.InvariantCulture), + state.Request.MediaSourceId, + user.GetToken()); + + builder.AppendFormat( + CultureInfo.InvariantCulture, + playlistFormat, + trickplayInfo.Bandwidth.ToString(CultureInfo.InvariantCulture), + trickplayInfo.Width.ToString(CultureInfo.InvariantCulture), + trickplayInfo.Height.ToString(CultureInfo.InvariantCulture), + url); + + builder.AppendLine(); + } + } + + /// <summary> /// Get the H.26X level of the output video stream. /// </summary> /// <param name="state">StreamState of the current stream.</param> diff --git a/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs b/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs index 60c529d4a..8548fec1a 100644 --- a/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs +++ b/Jellyfin.Api/Models/StreamingDtos/VideoRequestDto.cs @@ -1,4 +1,4 @@ -namespace Jellyfin.Api.Models.StreamingDtos; +namespace Jellyfin.Api.Models.StreamingDtos; /// <summary> /// The video request dto. @@ -15,4 +15,9 @@ public class VideoRequestDto : StreamingRequestDto /// Gets or sets a value indicating whether to enable subtitles in the manifest. /// </summary> public bool EnableSubtitlesInManifest { get; set; } + + /// <summary> + /// Gets or sets a value indicating whether to enable trickplay images. + /// </summary> + public bool EnableTrickplay { get; set; } } diff --git a/Jellyfin.Data/Entities/TrickplayInfo.cs b/Jellyfin.Data/Entities/TrickplayInfo.cs new file mode 100644 index 000000000..64e7da1b5 --- /dev/null +++ b/Jellyfin.Data/Entities/TrickplayInfo.cs @@ -0,0 +1,75 @@ +using System; +using System.Text.Json.Serialization; + +namespace Jellyfin.Data.Entities; + +/// <summary> +/// An entity representing the metadata for a group of trickplay tiles. +/// </summary> +public class TrickplayInfo +{ + /// <summary> + /// Gets or sets the id of the associated item. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + [JsonIgnore] + public Guid ItemId { get; set; } + + /// <summary> + /// Gets or sets width of an individual thumbnail. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int Width { get; set; } + + /// <summary> + /// Gets or sets height of an individual thumbnail. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int Height { get; set; } + + /// <summary> + /// Gets or sets amount of thumbnails per row. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int TileWidth { get; set; } + + /// <summary> + /// Gets or sets amount of thumbnails per column. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int TileHeight { get; set; } + + /// <summary> + /// Gets or sets total amount of non-black thumbnails. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int ThumbnailCount { get; set; } + + /// <summary> + /// Gets or sets interval in milliseconds between each trickplay thumbnail. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int Interval { get; set; } + + /// <summary> + /// Gets or sets peak bandwith usage in bits per second. + /// </summary> + /// <remarks> + /// Required. + /// </remarks> + public int Bandwidth { get; set; } +} diff --git a/Jellyfin.Server.Implementations/JellyfinDbContext.cs b/Jellyfin.Server.Implementations/JellyfinDbContext.cs index 0d91707e3..ea99af004 100644 --- a/Jellyfin.Server.Implementations/JellyfinDbContext.cs +++ b/Jellyfin.Server.Implementations/JellyfinDbContext.cs @@ -78,6 +78,11 @@ public class JellyfinDbContext : DbContext /// </summary> public DbSet<User> Users => Set<User>(); + /// <summary> + /// Gets the <see cref="DbSet{TEntity}"/> containing the trickplay metadata. + /// </summary> + public DbSet<TrickplayInfo> TrickplayInfos => Set<TrickplayInfo>(); + /*public DbSet<Artwork> Artwork => Set<Artwork>(); public DbSet<Book> Books => Set<Book>(); diff --git a/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.Designer.cs b/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.Designer.cs new file mode 100644 index 000000000..28baf1992 --- /dev/null +++ b/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.Designer.cs @@ -0,0 +1,681 @@ +// <auto-generated /> +using System; +using Jellyfin.Server.Implementations; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Jellyfin.Server.Implementations.Migrations +{ + [DbContext(typeof(JellyfinDbContext))] + [Migration("20230626233818_AddTrickplayInfos")] + partial class AddTrickplayInfos + { + /// <inheritdoc /> + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.7"); + + modelBuilder.Entity("Jellyfin.Data.Entities.AccessSchedule", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<int>("DayOfWeek") + .HasColumnType("INTEGER"); + + b.Property<double>("EndHour") + .HasColumnType("REAL"); + + b.Property<double>("StartHour") + .HasColumnType("REAL"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AccessSchedules"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.ActivityLog", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<DateTime>("DateCreated") + .HasColumnType("TEXT"); + + b.Property<string>("ItemId") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property<int>("LogSeverity") + .HasColumnType("INTEGER"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property<string>("Overview") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property<uint>("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property<string>("ShortOverview") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property<string>("Type") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("DateCreated"); + + b.ToTable("ActivityLogs"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.CustomItemDisplayPreferences", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<string>("Client") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<Guid>("ItemId") + .HasColumnType("TEXT"); + + b.Property<string>("Key") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.Property<string>("Value") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "ItemId", "Client", "Key") + .IsUnique(); + + b.ToTable("CustomItemDisplayPreferences"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.DisplayPreferences", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<int>("ChromecastVersion") + .HasColumnType("INTEGER"); + + b.Property<string>("Client") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<string>("DashboardTheme") + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<bool>("EnableNextVideoInfoOverlay") + .HasColumnType("INTEGER"); + + b.Property<int?>("IndexBy") + .HasColumnType("INTEGER"); + + b.Property<Guid>("ItemId") + .HasColumnType("TEXT"); + + b.Property<int>("ScrollDirection") + .HasColumnType("INTEGER"); + + b.Property<bool>("ShowBackdrop") + .HasColumnType("INTEGER"); + + b.Property<bool>("ShowSidebar") + .HasColumnType("INTEGER"); + + b.Property<int>("SkipBackwardLength") + .HasColumnType("INTEGER"); + + b.Property<int>("SkipForwardLength") + .HasColumnType("INTEGER"); + + b.Property<string>("TvHome") + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "ItemId", "Client") + .IsUnique(); + + b.ToTable("DisplayPreferences"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.HomeSection", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<int>("DisplayPreferencesId") + .HasColumnType("INTEGER"); + + b.Property<int>("Order") + .HasColumnType("INTEGER"); + + b.Property<int>("Type") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("DisplayPreferencesId"); + + b.ToTable("HomeSection"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.ImageInfo", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<DateTime>("LastModified") + .HasColumnType("TEXT"); + + b.Property<string>("Path") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property<Guid?>("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId") + .IsUnique(); + + b.ToTable("ImageInfos"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.ItemDisplayPreferences", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<string>("Client") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<int?>("IndexBy") + .HasColumnType("INTEGER"); + + b.Property<Guid>("ItemId") + .HasColumnType("TEXT"); + + b.Property<bool>("RememberIndexing") + .HasColumnType("INTEGER"); + + b.Property<bool>("RememberSorting") + .HasColumnType("INTEGER"); + + b.Property<string>("SortBy") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property<int>("SortOrder") + .HasColumnType("INTEGER"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.Property<int>("ViewType") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("ItemDisplayPreferences"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Permission", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<int>("Kind") + .HasColumnType("INTEGER"); + + b.Property<Guid?>("Permission_Permissions_Guid") + .HasColumnType("TEXT"); + + b.Property<uint>("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property<Guid?>("UserId") + .HasColumnType("TEXT"); + + b.Property<bool>("Value") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "Kind") + .IsUnique() + .HasFilter("[UserId] IS NOT NULL"); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Preference", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<int>("Kind") + .HasColumnType("INTEGER"); + + b.Property<Guid?>("Preference_Preferences_Guid") + .HasColumnType("TEXT"); + + b.Property<uint>("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property<Guid?>("UserId") + .HasColumnType("TEXT"); + + b.Property<string>("Value") + .IsRequired() + .HasMaxLength(65535) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "Kind") + .IsUnique() + .HasFilter("[UserId] IS NOT NULL"); + + b.ToTable("Preferences"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Security.ApiKey", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<string>("AccessToken") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property<DateTime>("DateCreated") + .HasColumnType("TEXT"); + + b.Property<DateTime>("DateLastActivity") + .HasColumnType("TEXT"); + + b.Property<string>("Name") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AccessToken") + .IsUnique(); + + b.ToTable("ApiKeys"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Security.Device", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<string>("AccessToken") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property<string>("AppName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property<string>("AppVersion") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property<DateTime>("DateCreated") + .HasColumnType("TEXT"); + + b.Property<DateTime>("DateLastActivity") + .HasColumnType("TEXT"); + + b.Property<DateTime>("DateModified") + .HasColumnType("TEXT"); + + b.Property<string>("DeviceId") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property<string>("DeviceName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property<bool>("IsActive") + .HasColumnType("INTEGER"); + + b.Property<Guid>("UserId") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("DeviceId"); + + b.HasIndex("AccessToken", "DateLastActivity"); + + b.HasIndex("DeviceId", "DateLastActivity"); + + b.HasIndex("UserId", "DeviceId"); + + b.ToTable("Devices"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Security.DeviceOptions", b => + { + b.Property<int>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property<string>("CustomName") + .HasColumnType("TEXT"); + + b.Property<string>("DeviceId") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("DeviceId") + .IsUnique(); + + b.ToTable("DeviceOptions"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.TrickplayInfo", b => + { + b.Property<Guid>("ItemId") + .HasColumnType("TEXT"); + + b.Property<int>("Width") + .HasColumnType("INTEGER"); + + b.Property<int>("Bandwidth") + .HasColumnType("INTEGER"); + + b.Property<int>("Height") + .HasColumnType("INTEGER"); + + b.Property<int>("Interval") + .HasColumnType("INTEGER"); + + b.Property<int>("ThumbnailCount") + .HasColumnType("INTEGER"); + + b.Property<int>("TileHeight") + .HasColumnType("INTEGER"); + + b.Property<int>("TileWidth") + .HasColumnType("INTEGER"); + + b.HasKey("ItemId", "Width"); + + b.ToTable("TrickplayInfos"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.User", b => + { + b.Property<Guid>("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property<string>("AudioLanguagePreference") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property<string>("AuthenticationProviderId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property<bool>("DisplayCollectionsView") + .HasColumnType("INTEGER"); + + b.Property<bool>("DisplayMissingEpisodes") + .HasColumnType("INTEGER"); + + b.Property<bool>("EnableAutoLogin") + .HasColumnType("INTEGER"); + + b.Property<bool>("EnableLocalPassword") + .HasColumnType("INTEGER"); + + b.Property<bool>("EnableNextEpisodeAutoPlay") + .HasColumnType("INTEGER"); + + b.Property<bool>("EnableUserPreferenceAccess") + .HasColumnType("INTEGER"); + + b.Property<bool>("HidePlayedInLatest") + .HasColumnType("INTEGER"); + + b.Property<long>("InternalId") + .HasColumnType("INTEGER"); + + b.Property<int>("InvalidLoginAttemptCount") + .HasColumnType("INTEGER"); + + b.Property<DateTime?>("LastActivityDate") + .HasColumnType("TEXT"); + + b.Property<DateTime?>("LastLoginDate") + .HasColumnType("TEXT"); + + b.Property<int?>("LoginAttemptsBeforeLockout") + .HasColumnType("INTEGER"); + + b.Property<int>("MaxActiveSessions") + .HasColumnType("INTEGER"); + + b.Property<int?>("MaxParentalAgeRating") + .HasColumnType("INTEGER"); + + b.Property<bool>("MustUpdatePassword") + .HasColumnType("INTEGER"); + + b.Property<string>("Password") + .HasMaxLength(65535) + .HasColumnType("TEXT"); + + b.Property<string>("PasswordResetProviderId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property<bool>("PlayDefaultAudioTrack") + .HasColumnType("INTEGER"); + + b.Property<bool>("RememberAudioSelections") + .HasColumnType("INTEGER"); + + b.Property<bool>("RememberSubtitleSelections") + .HasColumnType("INTEGER"); + + b.Property<int?>("RemoteClientBitrateLimit") + .HasColumnType("INTEGER"); + + b.Property<uint>("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property<string>("SubtitleLanguagePreference") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property<int>("SubtitleMode") + .HasColumnType("INTEGER"); + + b.Property<int>("SyncPlayAccess") + .HasColumnType("INTEGER"); + + b.Property<string>("Username") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT") + .UseCollation("NOCASE"); + + b.HasKey("Id"); + + b.HasIndex("Username") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.AccessSchedule", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithMany("AccessSchedules") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.DisplayPreferences", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithMany("DisplayPreferences") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.HomeSection", b => + { + b.HasOne("Jellyfin.Data.Entities.DisplayPreferences", null) + .WithMany("HomeSections") + .HasForeignKey("DisplayPreferencesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.ImageInfo", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithOne("ProfileImage") + .HasForeignKey("Jellyfin.Data.Entities.ImageInfo", "UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.ItemDisplayPreferences", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithMany("ItemDisplayPreferences") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Permission", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithMany("Permissions") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Preference", b => + { + b.HasOne("Jellyfin.Data.Entities.User", null) + .WithMany("Preferences") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.Security.Device", b => + { + b.HasOne("Jellyfin.Data.Entities.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.DisplayPreferences", b => + { + b.Navigation("HomeSections"); + }); + + modelBuilder.Entity("Jellyfin.Data.Entities.User", b => + { + b.Navigation("AccessSchedules"); + + b.Navigation("DisplayPreferences"); + + b.Navigation("ItemDisplayPreferences"); + + b.Navigation("Permissions"); + + b.Navigation("Preferences"); + + b.Navigation("ProfileImage"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.cs b/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.cs new file mode 100644 index 000000000..76b12de08 --- /dev/null +++ b/Jellyfin.Server.Implementations/Migrations/20230626233818_AddTrickplayInfos.cs @@ -0,0 +1,40 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Jellyfin.Server.Implementations.Migrations +{ + /// <inheritdoc /> + public partial class AddTrickplayInfos : Migration + { + /// <inheritdoc /> + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "TrickplayInfos", + columns: table => new + { + ItemId = table.Column<Guid>(type: "TEXT", nullable: false), + Width = table.Column<int>(type: "INTEGER", nullable: false), + Height = table.Column<int>(type: "INTEGER", nullable: false), + TileWidth = table.Column<int>(type: "INTEGER", nullable: false), + TileHeight = table.Column<int>(type: "INTEGER", nullable: false), + ThumbnailCount = table.Column<int>(type: "INTEGER", nullable: false), + Interval = table.Column<int>(type: "INTEGER", nullable: false), + Bandwidth = table.Column<int>(type: "INTEGER", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TrickplayInfos", x => new { x.ItemId, x.Width }); + }); + } + + /// <inheritdoc /> + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "TrickplayInfos"); + } + } +} diff --git a/Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs b/Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs index 6cd4594f6..f725ababe 100644 --- a/Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs +++ b/Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs @@ -1,4 +1,4 @@ -// <auto-generated /> +// <auto-generated /> using System; using Jellyfin.Server.Implementations; using Microsoft.EntityFrameworkCore; @@ -442,6 +442,37 @@ namespace Jellyfin.Server.Implementations.Migrations b.ToTable("DeviceOptions"); }); + modelBuilder.Entity("Jellyfin.Data.Entities.TrickplayInfo", b => + { + b.Property<Guid>("ItemId") + .HasColumnType("TEXT"); + + b.Property<int>("Width") + .HasColumnType("INTEGER"); + + b.Property<int>("Bandwidth") + .HasColumnType("INTEGER"); + + b.Property<int>("Height") + .HasColumnType("INTEGER"); + + b.Property<int>("Interval") + .HasColumnType("INTEGER"); + + b.Property<int>("ThumbnailCount") + .HasColumnType("INTEGER"); + + b.Property<int>("TileHeight") + .HasColumnType("INTEGER"); + + b.Property<int>("TileWidth") + .HasColumnType("INTEGER"); + + b.HasKey("ItemId", "Width"); + + b.ToTable("TrickplayInfos"); + }); + modelBuilder.Entity("Jellyfin.Data.Entities.User", b => { b.Property<Guid>("Id") diff --git a/Jellyfin.Server.Implementations/ModelConfiguration/TrickplayInfoConfiguration.cs b/Jellyfin.Server.Implementations/ModelConfiguration/TrickplayInfoConfiguration.cs new file mode 100644 index 000000000..dc1c17e5e --- /dev/null +++ b/Jellyfin.Server.Implementations/ModelConfiguration/TrickplayInfoConfiguration.cs @@ -0,0 +1,18 @@ +using Jellyfin.Data.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Jellyfin.Server.Implementations.ModelConfiguration +{ + /// <summary> + /// FluentAPI configuration for the TrickplayInfo entity. + /// </summary> + public class TrickplayInfoConfiguration : IEntityTypeConfiguration<TrickplayInfo> + { + /// <inheritdoc/> + public void Configure(EntityTypeBuilder<TrickplayInfo> builder) + { + builder.HasKey(info => new { info.ItemId, info.Width }); + } + } +} diff --git a/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs new file mode 100644 index 000000000..b960feb7f --- /dev/null +++ b/Jellyfin.Server.Implementations/Trickplay/TrickplayManager.cs @@ -0,0 +1,474 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Entities; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Drawing; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.MediaEncoding; +using MediaBrowser.Controller.Trickplay; +using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.IO; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Implementations.Trickplay; + +/// <summary> +/// ITrickplayManager implementation. +/// </summary> +public class TrickplayManager : ITrickplayManager +{ + private readonly ILogger<TrickplayManager> _logger; + private readonly IMediaEncoder _mediaEncoder; + private readonly IFileSystem _fileSystem; + private readonly EncodingHelper _encodingHelper; + private readonly ILibraryManager _libraryManager; + private readonly IServerConfigurationManager _config; + private readonly IImageEncoder _imageEncoder; + private readonly IDbContextFactory<JellyfinDbContext> _dbProvider; + private readonly IApplicationPaths _appPaths; + + private static readonly SemaphoreSlim _resourcePool = new(1, 1); + private static readonly string[] _trickplayImgExtensions = { ".jpg" }; + + /// <summary> + /// Initializes a new instance of the <see cref="TrickplayManager"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="mediaEncoder">The media encoder.</param> + /// <param name="fileSystem">The file systen.</param> + /// <param name="encodingHelper">The encoding helper.</param> + /// <param name="libraryManager">The library manager.</param> + /// <param name="config">The server configuration manager.</param> + /// <param name="imageEncoder">The image encoder.</param> + /// <param name="dbProvider">The database provider.</param> + /// <param name="appPaths">The application paths.</param> + public TrickplayManager( + ILogger<TrickplayManager> logger, + IMediaEncoder mediaEncoder, + IFileSystem fileSystem, + EncodingHelper encodingHelper, + ILibraryManager libraryManager, + IServerConfigurationManager config, + IImageEncoder imageEncoder, + IDbContextFactory<JellyfinDbContext> dbProvider, + IApplicationPaths appPaths) + { + _logger = logger; + _mediaEncoder = mediaEncoder; + _fileSystem = fileSystem; + _encodingHelper = encodingHelper; + _libraryManager = libraryManager; + _config = config; + _imageEncoder = imageEncoder; + _dbProvider = dbProvider; + _appPaths = appPaths; + } + + /// <inheritdoc /> + public async Task RefreshTrickplayDataAsync(Video video, bool replace, CancellationToken cancellationToken) + { + _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace); + + var options = _config.Configuration.TrickplayOptions; + foreach (var width in options.WidthResolutions) + { + cancellationToken.ThrowIfCancellationRequested(); + await RefreshTrickplayDataInternal( + video, + replace, + width, + options, + cancellationToken).ConfigureAwait(false); + } + } + + private async Task RefreshTrickplayDataInternal( + Video video, + bool replace, + int width, + TrickplayOptions options, + CancellationToken cancellationToken) + { + if (!CanGenerateTrickplay(video, options.Interval)) + { + return; + } + + var imgTempDir = string.Empty; + var outputDir = GetTrickplayDirectory(video, width); + + await _resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + if (!replace && Directory.Exists(outputDir) && (await GetTrickplayResolutions(video.Id).ConfigureAwait(false)).ContainsKey(width)) + { + _logger.LogDebug("Found existing trickplay files for {ItemId}. Exiting.", video.Id); + return; + } + + // Extract images + // Note: Media sources under parent items exist as their own video/item as well. Only use this video stream for trickplay. + var mediaSource = video.GetMediaSources(false).Find(source => Guid.Parse(source.Id).Equals(video.Id)); + + if (mediaSource is null) + { + _logger.LogDebug("Found no matching media source for item {ItemId}", video.Id); + return; + } + + var mediaPath = mediaSource.Path; + var mediaStream = mediaSource.VideoStream; + var container = mediaSource.Container; + + _logger.LogInformation("Creating trickplay files at {Width} width, for {Path} [ID: {ItemId}]", width, mediaPath, video.Id); + imgTempDir = await _mediaEncoder.ExtractVideoImagesOnIntervalAccelerated( + mediaPath, + container, + mediaSource, + mediaStream, + width, + TimeSpan.FromMilliseconds(options.Interval), + options.EnableHwAcceleration, + options.ProcessThreads, + options.Qscale, + options.ProcessPriority, + _encodingHelper, + cancellationToken).ConfigureAwait(false); + + if (string.IsNullOrEmpty(imgTempDir) || !Directory.Exists(imgTempDir)) + { + throw new InvalidOperationException("Null or invalid directory from media encoder."); + } + + var images = _fileSystem.GetFiles(imgTempDir, _trickplayImgExtensions, false, false) + .Select(i => i.FullName) + .OrderBy(i => i) + .ToList(); + + // Create tiles + var trickplayInfo = CreateTiles(images, width, options, outputDir); + + // Save tiles info + try + { + if (trickplayInfo is not null) + { + trickplayInfo.ItemId = video.Id; + await SaveTrickplayInfo(trickplayInfo).ConfigureAwait(false); + + _logger.LogInformation("Finished creation of trickplay files for {0}", mediaPath); + } + else + { + throw new InvalidOperationException("Null trickplay tiles info from CreateTiles."); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error while saving trickplay tiles info."); + + // Make sure no files stay in metadata folders on failure + // if tiles info wasn't saved. + Directory.Delete(outputDir, true); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Error creating trickplay images."); + } + finally + { + _resourcePool.Release(); + + if (!string.IsNullOrEmpty(imgTempDir)) + { + Directory.Delete(imgTempDir, true); + } + } + } + + /// <inheritdoc /> + public TrickplayInfo CreateTiles(List<string> images, int width, TrickplayOptions options, string outputDir) + { + if (images.Count == 0) + { + throw new ArgumentException("Can't create trickplay from 0 images."); + } + + var workDir = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(workDir); + + var trickplayInfo = new TrickplayInfo + { + Width = width, + Interval = options.Interval, + TileWidth = options.TileWidth, + TileHeight = options.TileHeight, + ThumbnailCount = images.Count, + // Set during image generation + Height = 0, + Bandwidth = 0 + }; + + /* + * Generate trickplay tiles from sets of thumbnails + */ + var imageOptions = new ImageCollageOptions + { + Width = trickplayInfo.TileWidth, + Height = trickplayInfo.TileHeight + }; + + var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight; + var requiredTiles = (int)Math.Ceiling((double)images.Count / thumbnailsPerTile); + + for (int i = 0; i < requiredTiles; i++) + { + // Set output/input paths + var tilePath = Path.Combine(workDir, $"{i}.jpg"); + + imageOptions.OutputPath = tilePath; + imageOptions.InputPaths = images.GetRange(i * thumbnailsPerTile, Math.Min(thumbnailsPerTile, images.Count - (i * thumbnailsPerTile))); + + // Generate image and use returned height for tiles info + var height = _imageEncoder.CreateTrickplayTile(imageOptions, options.JpegQuality, trickplayInfo.Width, trickplayInfo.Height != 0 ? trickplayInfo.Height : null); + if (trickplayInfo.Height == 0) + { + trickplayInfo.Height = height; + } + + // Update bitrate + var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tilePath).Length * 8 / trickplayInfo.TileWidth / trickplayInfo.TileHeight / (trickplayInfo.Interval / 1000)); + trickplayInfo.Bandwidth = Math.Max(trickplayInfo.Bandwidth, bitrate); + } + + /* + * Move trickplay tiles to output directory + */ + Directory.CreateDirectory(Directory.GetParent(outputDir)!.FullName); + + // Replace existing tiles if they already exist + if (Directory.Exists(outputDir)) + { + Directory.Delete(outputDir, true); + } + + MoveDirectory(workDir, outputDir); + + return trickplayInfo; + } + + private bool CanGenerateTrickplay(Video video, int interval) + { + var videoType = video.VideoType; + if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay) + { + return false; + } + + if (video.IsPlaceHolder) + { + return false; + } + + if (video.IsShortcut) + { + return false; + } + + if (!video.IsCompleteMedia) + { + return false; + } + + if (!video.RunTimeTicks.HasValue || video.RunTimeTicks.Value < TimeSpan.FromMilliseconds(interval).Ticks) + { + return false; + } + + var libraryOptions = _libraryManager.GetLibraryOptions(video); + if (libraryOptions is null || !libraryOptions.EnableTrickplayImageExtraction) + { + return false; + } + + // Can't extract images if there are no video streams + return video.GetMediaStreams().Count > 0; + } + + /// <inheritdoc /> + public async Task<Dictionary<int, TrickplayInfo>> GetTrickplayResolutions(Guid itemId) + { + var trickplayResolutions = new Dictionary<int, TrickplayInfo>(); + + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var trickplayInfos = await dbContext.TrickplayInfos + .AsNoTracking() + .Where(i => i.ItemId.Equals(itemId)) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var info in trickplayInfos) + { + trickplayResolutions[info.Width] = info; + } + } + + return trickplayResolutions; + } + + /// <inheritdoc /> + public async Task SaveTrickplayInfo(TrickplayInfo info) + { + var dbContext = await _dbProvider.CreateDbContextAsync().ConfigureAwait(false); + await using (dbContext.ConfigureAwait(false)) + { + var oldInfo = await dbContext.TrickplayInfos.FindAsync(info.ItemId, info.Width).ConfigureAwait(false); + if (oldInfo is not null) + { + dbContext.TrickplayInfos.Remove(oldInfo); + } + + dbContext.Add(info); + + await dbContext.SaveChangesAsync().ConfigureAwait(false); + } + } + + /// <inheritdoc /> + public async Task<Dictionary<string, Dictionary<int, TrickplayInfo>>> GetTrickplayManifest(BaseItem item) + { + var trickplayManifest = new Dictionary<string, Dictionary<int, TrickplayInfo>>(); + foreach (var mediaSource in item.GetMediaSources(false)) + { + var mediaSourceId = Guid.Parse(mediaSource.Id); + var trickplayResolutions = await GetTrickplayResolutions(mediaSourceId).ConfigureAwait(false); + + if (trickplayResolutions.Count > 0) + { + trickplayManifest[mediaSource.Id] = trickplayResolutions; + } + } + + return trickplayManifest; + } + + /// <inheritdoc /> + public string GetTrickplayTilePath(BaseItem item, int width, int index) + { + return Path.Combine(GetTrickplayDirectory(item, width), index + ".jpg"); + } + + /// <inheritdoc /> + public async Task<string?> GetHlsPlaylist(Guid itemId, int width, string? apiKey) + { + var trickplayResolutions = await GetTrickplayResolutions(itemId).ConfigureAwait(false); + if (trickplayResolutions is not null && trickplayResolutions.TryGetValue(width, out var trickplayInfo)) + { + var builder = new StringBuilder(128); + + if (trickplayInfo.ThumbnailCount > 0) + { + const string urlFormat = "Trickplay/{0}/{1}.jpg?MediaSourceId={2}&api_key={3}"; + const string decimalFormat = "{0:0.###}"; + + var resolution = $"{trickplayInfo.Width}x{trickplayInfo.Height}"; + var layout = $"{trickplayInfo.TileWidth}x{trickplayInfo.TileHeight}"; + var thumbnailsPerTile = trickplayInfo.TileWidth * trickplayInfo.TileHeight; + var thumbnailDuration = trickplayInfo.Interval / 1000d; + var infDuration = thumbnailDuration * thumbnailsPerTile; + var tileCount = (int)Math.Ceiling((decimal)trickplayInfo.ThumbnailCount / thumbnailsPerTile); + + builder + .AppendLine("#EXTM3U") + .Append("#EXT-X-TARGETDURATION:") + .AppendLine(tileCount.ToString(CultureInfo.InvariantCulture)) + .AppendLine("#EXT-X-VERSION:7") + .AppendLine("#EXT-X-MEDIA-SEQUENCE:1") + .AppendLine("#EXT-X-PLAYLIST-TYPE:VOD") + .AppendLine("#EXT-X-IMAGES-ONLY"); + + for (int i = 0; i < tileCount; i++) + { + // All tiles prior to the last must contain full amount of thumbnails (no black). + if (i == tileCount - 1) + { + thumbnailsPerTile = trickplayInfo.ThumbnailCount - (i * thumbnailsPerTile); + infDuration = thumbnailDuration * thumbnailsPerTile; + } + + // EXTINF + builder + .Append("#EXTINF:") + .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, infDuration) + .AppendLine(","); + + // EXT-X-TILES + builder + .Append("#EXT-X-TILES:RESOLUTION=") + .Append(resolution) + .Append(",LAYOUT=") + .Append(layout) + .Append(",DURATION=") + .AppendFormat(CultureInfo.InvariantCulture, decimalFormat, thumbnailDuration) + .AppendLine(); + + // URL + builder + .AppendFormat( + CultureInfo.InvariantCulture, + urlFormat, + width.ToString(CultureInfo.InvariantCulture), + i.ToString(CultureInfo.InvariantCulture), + itemId.ToString("N"), + apiKey) + .AppendLine(); + } + + builder.AppendLine("#EXT-X-ENDLIST"); + return builder.ToString(); + } + } + + return null; + } + + private string GetTrickplayDirectory(BaseItem item, int? width = null) + { + var path = Path.Combine(item.GetInternalMetadataPath(), "trickplay"); + + return width.HasValue ? Path.Combine(path, width.Value.ToString(CultureInfo.InvariantCulture)) : path; + } + + private void MoveDirectory(string source, string destination) + { + try + { + Directory.Move(source, destination); + } + catch (IOException) + { + // Cross device move requires a copy + Directory.CreateDirectory(destination); + foreach (string file in Directory.GetFiles(source)) + { + File.Copy(file, Path.Join(destination, Path.GetFileName(file)), true); + } + + Directory.Delete(source, true); + } + } +} diff --git a/Jellyfin.Server/CoreAppHost.cs b/Jellyfin.Server/CoreAppHost.cs index 4c116745b..c12c90a68 100644 --- a/Jellyfin.Server/CoreAppHost.cs +++ b/Jellyfin.Server/CoreAppHost.cs @@ -11,6 +11,7 @@ using Jellyfin.Server.Implementations.Activity; using Jellyfin.Server.Implementations.Devices; using Jellyfin.Server.Implementations.Events; using Jellyfin.Server.Implementations.Security; +using Jellyfin.Server.Implementations.Trickplay; using Jellyfin.Server.Implementations.Users; using MediaBrowser.Controller; using MediaBrowser.Controller.BaseItemManager; @@ -21,6 +22,7 @@ using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Lyrics; using MediaBrowser.Controller.Net; using MediaBrowser.Controller.Security; +using MediaBrowser.Controller.Trickplay; using MediaBrowser.Model.Activity; using MediaBrowser.Providers.Lyric; using Microsoft.Extensions.Configuration; @@ -78,6 +80,7 @@ namespace Jellyfin.Server serviceCollection.AddSingleton<IUserManager, UserManager>(); serviceCollection.AddScoped<IDisplayPreferencesManager, DisplayPreferencesManager>(); serviceCollection.AddSingleton<IDeviceManager, DeviceManager>(); + serviceCollection.AddSingleton<ITrickplayManager, TrickplayManager>(); // TODO search the assemblies instead of adding them manually? serviceCollection.AddSingleton<IWebSocketListener, SessionWebSocketListener>(); diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index e5c8ebfaf..c7bfbdb53 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -81,5 +81,15 @@ namespace MediaBrowser.Controller.Drawing /// <param name="posters">The list of poster paths.</param> /// <param name="backdrops">The list of backdrop paths.</param> void CreateSplashscreen(IReadOnlyList<string> posters, IReadOnlyList<string> backdrops); + + /// <summary> + /// Creates a new trickplay tile image. + /// </summary> + /// <param name="options">The options to use when creating the image. Width and Height are a quantity of thumbnails in this case, not pixels.</param> + /// <param name="quality">The image encode quality.</param> + /// <param name="imgWidth">The width of a single trickplay thumbnail.</param> + /// <param name="imgHeight">Optional height of a single trickplay thumbnail, if it is known.</param> + /// <returns>Height of single decoded trickplay thumbnail.</returns> + int CreateTrickplayTile(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight); } } diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 08ce19f69..c3a20cdb4 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -100,6 +100,13 @@ namespace MediaBrowser.Controller.MediaEncoding { "truehd", 6 }, }; + private static readonly string _defaultMjpegEncoder = "mjpeg"; + private static readonly Dictionary<string, string> _mjpegCodecMap = new(StringComparer.OrdinalIgnoreCase) + { + { "vaapi", _defaultMjpegEncoder + "_vaapi" }, + { "qsv", _defaultMjpegEncoder + "_qsv" } + }; + public static readonly string[] LosslessAudioCodecs = new string[] { "alac", @@ -167,6 +174,24 @@ namespace MediaBrowser.Controller.MediaEncoding return defaultEncoder; } + private string GetMjpegEncoder(EncodingJobInfo state, EncodingOptions encodingOptions) + { + if (state.VideoType == VideoType.VideoFile) + { + var hwType = encodingOptions.HardwareAccelerationType; + + if (!string.IsNullOrEmpty(hwType) + && encodingOptions.EnableHardwareEncoding + && _mjpegCodecMap.TryGetValue(hwType, out var preferredEncoder) + && _mediaEncoder.SupportsEncoder(preferredEncoder)) + { + return preferredEncoder; + } + } + + return _defaultMjpegEncoder; + } + private bool IsVaapiSupported(EncodingJobInfo state) { // vaapi will throw an error with this input @@ -300,6 +325,11 @@ namespace MediaBrowser.Controller.MediaEncoding return GetH264Encoder(state, encodingOptions); } + if (string.Equals(codec, "mjpeg", StringComparison.OrdinalIgnoreCase)) + { + return GetMjpegEncoder(state, encodingOptions); + } + if (string.Equals(codec, "vp8", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "vpx", StringComparison.OrdinalIgnoreCase)) { @@ -4917,6 +4947,15 @@ namespace MediaBrowser.Controller.MediaEncoding subFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); overlayFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + var framerate = GetFramerateParam(state); + if (framerate.HasValue) + { + mainFilters.Insert(0, string.Format( + CultureInfo.InvariantCulture, + "fps={0}", + framerate.Value)); + } + var mainStr = string.Empty; if (mainFilters?.Count > 0) { diff --git a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs index 4114dea4f..c2cef4978 100644 --- a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs +++ b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs @@ -4,8 +4,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; @@ -138,6 +140,36 @@ namespace MediaBrowser.Controller.MediaEncoding Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, ImageFormat? targetFormat, CancellationToken cancellationToken); /// <summary> + /// Extracts the video images on interval. + /// </summary> + /// <param name="inputFile">Input file.</param> + /// <param name="container">Video container type.</param> + /// <param name="mediaSource">Media source information.</param> + /// <param name="imageStream">Media stream information.</param> + /// <param name="maxWidth">The maximum width.</param> + /// <param name="interval">The interval.</param> + /// <param name="allowHwAccel">Allow for hardware acceleration.</param> + /// <param name="threads">The input/output thread count for ffmpeg.</param> + /// <param name="qualityScale">The qscale value for ffmpeg.</param> + /// <param name="priority">The process priority for the ffmpeg process.</param> + /// <param name="encodingHelper">EncodingHelper instance.</param> + /// <param name="cancellationToken">The cancellation token.</param> + /// <returns>Directory where images where extracted. A given image made before another will always be named with a lower number.</returns> + Task<string> ExtractVideoImagesOnIntervalAccelerated( + string inputFile, + string container, + MediaSourceInfo mediaSource, + MediaStream imageStream, + int maxWidth, + TimeSpan interval, + bool allowHwAccel, + int? threads, + int? qualityScale, + ProcessPriorityClass? priority, + EncodingHelper encodingHelper, + CancellationToken cancellationToken); + + /// <summary> /// Gets the media info. /// </summary> /// <param name="request">The request.</param> diff --git a/MediaBrowser.Controller/Trickplay/ITrickplayManager.cs b/MediaBrowser.Controller/Trickplay/ITrickplayManager.cs new file mode 100644 index 000000000..0c41f3023 --- /dev/null +++ b/MediaBrowser.Controller/Trickplay/ITrickplayManager.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Data.Entities; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Model.Configuration; + +namespace MediaBrowser.Controller.Trickplay; + +/// <summary> +/// Interface ITrickplayManager. +/// </summary> +public interface ITrickplayManager +{ + /// <summary> + /// Generates new trickplay images and metadata. + /// </summary> + /// <param name="video">The video.</param> + /// <param name="replace">Whether or not existing data should be replaced.</param> + /// <param name="cancellationToken">CancellationToken to use for operation.</param> + /// <returns>Task.</returns> + Task RefreshTrickplayDataAsync(Video video, bool replace, CancellationToken cancellationToken); + + /// <summary> + /// Creates trickplay tiles out of individual thumbnails. + /// </summary> + /// <param name="images">Ordered file paths of the thumbnails to be used.</param> + /// <param name="width">The width of a single thumbnail.</param> + /// <param name="options">The trickplay options.</param> + /// <param name="outputDir">The output directory.</param> + /// <returns>The associated trickplay information.</returns> + /// <remarks> + /// The output directory will be DELETED and replaced if it already exists. + /// </remarks> + TrickplayInfo CreateTiles(List<string> images, int width, TrickplayOptions options, string outputDir); + + /// <summary> + /// Get available trickplay resolutions and corresponding info. + /// </summary> + /// <param name="itemId">The item.</param> + /// <returns>Map of width resolutions to trickplay tiles info.</returns> + Task<Dictionary<int, TrickplayInfo>> GetTrickplayResolutions(Guid itemId); + + /// <summary> + /// Saves trickplay info. + /// </summary> + /// <param name="info">The trickplay info.</param> + /// <returns>Task.</returns> + Task SaveTrickplayInfo(TrickplayInfo info); + + /// <summary> + /// Gets all trickplay infos for all media streams of an item. + /// </summary> + /// <param name="item">The item.</param> + /// <returns>A map of media source id to a map of tile width to trickplay info.</returns> + Task<Dictionary<string, Dictionary<int, TrickplayInfo>>> GetTrickplayManifest(BaseItem item); + + /// <summary> + /// Gets the path to a trickplay tile image. + /// </summary> + /// <param name="item">The item.</param> + /// <param name="width">The width of a single thumbnail.</param> + /// <param name="index">The tile's index.</param> + /// <returns>The absolute path.</returns> + string GetTrickplayTilePath(BaseItem item, int width, int index); + + /// <summary> + /// Gets the trickplay HLS playlist. + /// </summary> + /// <param name="itemId">The item.</param> + /// <param name="width">The width of a single thumbnail.</param> + /// <param name="apiKey">Optional api key of the requesting user.</param> + /// <returns>The text content of the .m3u8 playlist.</returns> + Task<string?> GetHlsPlaylist(Guid itemId, int width, string? apiKey); +} diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 629f8afde..4668b8bbb 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -21,6 +21,7 @@ using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Extensions; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.MediaEncoding.Probing; +using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; @@ -28,8 +29,10 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Globalization; using MediaBrowser.Model.IO; using MediaBrowser.Model.MediaInfo; +using Microsoft.AspNetCore.Components.Forms; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; +using static Nikse.SubtitleEdit.Core.Common.IfoParser; namespace MediaBrowser.MediaEncoding.Encoder { @@ -781,6 +784,191 @@ namespace MediaBrowser.MediaEncoding.Encoder } /// <inheritdoc /> + public Task<string> ExtractVideoImagesOnIntervalAccelerated( + string inputFile, + string container, + MediaSourceInfo mediaSource, + MediaStream imageStream, + int maxWidth, + TimeSpan interval, + bool allowHwAccel, + int? threads, + int? qualityScale, + ProcessPriorityClass? priority, + EncodingHelper encodingHelper, + CancellationToken cancellationToken) + { + var options = allowHwAccel ? _configurationManager.GetEncodingOptions() : new EncodingOptions(); + threads ??= _threads; + + // A new EncodingOptions instance must be used as to not disable HW acceleration for all of Jellyfin. + // Additionally, we must set a few fields without defaults to prevent null pointer exceptions. + if (!allowHwAccel) + { + options.EnableHardwareEncoding = false; + options.HardwareAccelerationType = string.Empty; + options.EnableTonemapping = false; + } + + var baseRequest = new BaseEncodingJobOptions { MaxWidth = maxWidth, MaxFramerate = (float)(1.0 / interval.TotalSeconds) }; + var jobState = new EncodingJobInfo(TranscodingJobType.Progressive) + { + IsVideoRequest = true, // must be true for InputVideoHwaccelArgs to return non-empty value + MediaSource = mediaSource, + VideoStream = imageStream, + BaseRequest = baseRequest, // GetVideoProcessingFilterParam errors if null + MediaPath = inputFile, + OutputVideoCodec = "mjpeg" + }; + var vidEncoder = options.AllowMjpegEncoding ? encodingHelper.GetVideoEncoder(jobState, options) : jobState.OutputVideoCodec; + + // Get input and filter arguments + var inputArg = encodingHelper.GetInputArgument(jobState, options, container).Trim(); + if (string.IsNullOrWhiteSpace(inputArg)) + { + throw new InvalidOperationException("EncodingHelper returned empty input arguments."); + } + + if (!allowHwAccel) + { + inputArg = "-threads " + threads + " " + inputArg; // HW accel args set a different input thread count, only set if disabled + } + + var filterParam = encodingHelper.GetVideoProcessingFilterParam(jobState, options, jobState.OutputVideoCodec).Trim(); + if (string.IsNullOrWhiteSpace(filterParam)) + { + throw new InvalidOperationException("EncodingHelper returned empty or invalid filter parameters."); + } + + return ExtractVideoImagesOnIntervalInternal(inputArg, filterParam, vidEncoder, threads, qualityScale, priority, cancellationToken); + } + + private async Task<string> ExtractVideoImagesOnIntervalInternal( + string inputArg, + string filterParam, + string vidEncoder, + int? outputThreads, + int? qualityScale, + ProcessPriorityClass? priority, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(inputArg)) + { + throw new InvalidOperationException("Empty or invalid input argument."); + } + + // Output arguments + var targetDirectory = Path.Combine(_configurationManager.ApplicationPaths.TempDirectory, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(targetDirectory); + var outputPath = Path.Combine(targetDirectory, "%08d.jpg"); + + // Final command arguments + var args = string.Format( + CultureInfo.InvariantCulture, + "-loglevel error {0} -an -sn {1} -threads {2} -c:v {3} {4}-f {5} \"{6}\"", + inputArg, + filterParam, + outputThreads.GetValueOrDefault(_threads), + vidEncoder, + qualityScale.HasValue ? "-qscale:v " + qualityScale.Value.ToString(CultureInfo.InvariantCulture) + " " : string.Empty, + "image2", + outputPath); + + // Start ffmpeg process + var process = new Process + { + StartInfo = new ProcessStartInfo + { + CreateNoWindow = true, + UseShellExecute = false, + FileName = _ffmpegPath, + Arguments = args, + WindowStyle = ProcessWindowStyle.Hidden, + ErrorDialog = false, + }, + EnableRaisingEvents = true + }; + + var processDescription = string.Format(CultureInfo.InvariantCulture, "{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments); + _logger.LogInformation("Trickplay generation: {ProcessDescription}", processDescription); + + using (var processWrapper = new ProcessWrapper(process, this)) + { + bool ranToCompletion = false; + + await _thumbnailResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + StartProcess(processWrapper); + + // Set process priority + if (priority.HasValue) + { + try + { + processWrapper.Process.PriorityClass = priority.Value; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Unable to set process priority to {Priority} for {Description}", priority.Value, processDescription); + } + } + + // Need to give ffmpeg enough time to make all the thumbnails, which could be a while, + // but we still need to detect if the process hangs. + // Making the assumption that as long as new jpegs are showing up, everything is good. + + bool isResponsive = true; + int lastCount = 0; + var timeoutMs = _configurationManager.Configuration.ImageExtractionTimeoutMs; + timeoutMs = timeoutMs <= 0 ? DefaultHdrImageExtractionTimeout : timeoutMs; + + while (isResponsive) + { + try + { + await process.WaitForExitAsync(TimeSpan.FromMilliseconds(timeoutMs)).ConfigureAwait(false); + + ranToCompletion = true; + break; + } + catch (OperationCanceledException) + { + // We don't actually expect the process to be finished in one timeout span, just that one image has been generated. + } + + cancellationToken.ThrowIfCancellationRequested(); + + var jpegCount = _fileSystem.GetFilePaths(targetDirectory).Count(); + + isResponsive = jpegCount > lastCount; + lastCount = jpegCount; + } + + if (!ranToCompletion) + { + _logger.LogInformation("Stopping trickplay extraction due to process inactivity."); + StopProcess(processWrapper, 1000); + } + } + finally + { + _thumbnailResourcePool.Release(); + } + + var exitCode = ranToCompletion ? processWrapper.ExitCode ?? 0 : -1; + + if (exitCode == -1) + { + _logger.LogError("ffmpeg image extraction failed for {ProcessDescription}", processDescription); + + throw new FfmpegException(string.Format(CultureInfo.InvariantCulture, "ffmpeg image extraction failed for {0}", processDescription)); + } + + return targetDirectory; + } + } + public string GetTimeParameter(long ticks) { var time = TimeSpan.FromTicks(ticks); diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs index 3f0e98ec8..84c735f9c 100644 --- a/MediaBrowser.Model/Configuration/EncodingOptions.cs +++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs @@ -50,6 +50,7 @@ public class EncodingOptions EnableHardwareEncoding = true; AllowHevcEncoding = false; AllowAv1Encoding = false; + AllowMjpegEncoding = false; EnableSubtitleExtraction = true; AllowOnDemandMetadataBasedKeyframeExtractionForExtensions = new[] { "mkv" }; HardwareDecodingCodecs = new string[] { "h264", "vc1" }; @@ -256,6 +257,11 @@ public class EncodingOptions public bool AllowAv1Encoding { get; set; } /// <summary> + /// Gets or sets a value indicating whether MJPEG encoding is enabled. + /// </summary> + public bool AllowMjpegEncoding { get; set; } + + /// <summary> /// Gets or sets a value indicating whether subtitle extraction is enabled. /// </summary> public bool EnableSubtitleExtraction { get; set; } diff --git a/MediaBrowser.Model/Configuration/LibraryOptions.cs b/MediaBrowser.Model/Configuration/LibraryOptions.cs index 9743edb1c..fbad29143 100644 --- a/MediaBrowser.Model/Configuration/LibraryOptions.cs +++ b/MediaBrowser.Model/Configuration/LibraryOptions.cs @@ -35,6 +35,10 @@ namespace MediaBrowser.Model.Configuration public bool ExtractChapterImagesDuringLibraryScan { get; set; } + public bool EnableTrickplayImageExtraction { get; set; } + + public bool ExtractTrickplayImagesDuringLibraryScan { get; set; } + public MediaPathInfo[] PathInfos { get; set; } public bool SaveLocalMetadata { get; set; } diff --git a/MediaBrowser.Model/Configuration/ServerConfiguration.cs b/MediaBrowser.Model/Configuration/ServerConfiguration.cs index 1342ffb48..1c9cc6c01 100644 --- a/MediaBrowser.Model/Configuration/ServerConfiguration.cs +++ b/MediaBrowser.Model/Configuration/ServerConfiguration.cs @@ -270,4 +270,10 @@ public class ServerConfiguration : BaseApplicationConfiguration /// Gets or sets the list of cast receiver applications. /// </summary> public CastReceiverApplication[] CastReceiverApplications { get; set; } = Array.Empty<CastReceiverApplication>(); + + /// <summary> + /// Gets or sets the trickplay options. + /// </summary> + /// <value>The trickplay options.</value> + public TrickplayOptions TrickplayOptions { get; set; } = new TrickplayOptions(); } diff --git a/MediaBrowser.Model/Configuration/TrickplayOptions.cs b/MediaBrowser.Model/Configuration/TrickplayOptions.cs new file mode 100644 index 000000000..92c16ee84 --- /dev/null +++ b/MediaBrowser.Model/Configuration/TrickplayOptions.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Diagnostics; + +namespace MediaBrowser.Model.Configuration; + +/// <summary> +/// Class TrickplayOptions. +/// </summary> +public class TrickplayOptions +{ + /// <summary> + /// Gets or sets a value indicating whether or not to use HW acceleration. + /// </summary> + public bool EnableHwAcceleration { get; set; } = false; + + /// <summary> + /// Gets or sets the behavior used by trickplay provider on library scan/update. + /// </summary> + public TrickplayScanBehavior ScanBehavior { get; set; } = TrickplayScanBehavior.NonBlocking; + + /// <summary> + /// Gets or sets the process priority for the ffmpeg process. + /// </summary> + public ProcessPriorityClass ProcessPriority { get; set; } = ProcessPriorityClass.BelowNormal; + + /// <summary> + /// Gets or sets the interval, in ms, between each new trickplay image. + /// </summary> + public int Interval { get; set; } = 10000; + + /// <summary> + /// Gets or sets the target width resolutions, in px, to generates preview images for. + /// </summary> + public int[] WidthResolutions { get; set; } = new[] { 320 }; + + /// <summary> + /// Gets or sets number of tile images to allow in X dimension. + /// </summary> + public int TileWidth { get; set; } = 10; + + /// <summary> + /// Gets or sets number of tile images to allow in Y dimension. + /// </summary> + public int TileHeight { get; set; } = 10; + + /// <summary> + /// Gets or sets the ffmpeg output quality level. + /// </summary> + public int Qscale { get; set; } = 4; + + /// <summary> + /// Gets or sets the jpeg quality to use for image tiles. + /// </summary> + public int JpegQuality { get; set; } = 90; + + /// <summary> + /// Gets or sets the number of threads to be used by ffmpeg. + /// </summary> + public int ProcessThreads { get; set; } = 1; +} diff --git a/MediaBrowser.Model/Configuration/TrickplayScanBehavior.cs b/MediaBrowser.Model/Configuration/TrickplayScanBehavior.cs new file mode 100644 index 000000000..d0db53218 --- /dev/null +++ b/MediaBrowser.Model/Configuration/TrickplayScanBehavior.cs @@ -0,0 +1,17 @@ +namespace MediaBrowser.Model.Configuration; + +/// <summary> +/// Enum TrickplayScanBehavior. +/// </summary> +public enum TrickplayScanBehavior +{ + /// <summary> + /// Starts generation, only return once complete. + /// </summary> + Blocking, + + /// <summary> + /// Start generation, return immediately. + /// </summary> + NonBlocking +} diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 8fab1ca6d..287966dd0 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Jellyfin.Data.Entities; using Jellyfin.Data.Enums; using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Entities; @@ -569,6 +570,12 @@ namespace MediaBrowser.Model.Dto public List<ChapterInfo> Chapters { get; set; } /// <summary> + /// Gets or sets the trickplay manifest. + /// </summary> + /// <value>The trickplay manifest.</value> + public Dictionary<string, Dictionary<int, TrickplayInfo>> Trickplay { get; set; } + + /// <summary> /// Gets or sets the type of the location. /// </summary> /// <value>The type of the location.</value> diff --git a/MediaBrowser.Model/Querying/ItemFields.cs b/MediaBrowser.Model/Querying/ItemFields.cs index 6fa1d778a..242a1c6e9 100644 --- a/MediaBrowser.Model/Querying/ItemFields.cs +++ b/MediaBrowser.Model/Querying/ItemFields.cs @@ -34,6 +34,11 @@ namespace MediaBrowser.Model.Querying /// </summary> Chapters, + /// <summary> + /// The trickplay manifest. + /// </summary> + Trickplay, + ChildCount, /// <summary> diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index 6a40833d7..7ef70f4b0 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -1,4 +1,4 @@ -<Project Sdk="Microsoft.NET.Sdk"> +<Project Sdk="Microsoft.NET.Sdk"> <!-- ProjectGuid is only included as a requirement for SonarQube analysis --> <PropertyGroup> diff --git a/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs new file mode 100644 index 000000000..69f10b43b --- /dev/null +++ b/MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Trickplay; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Globalization; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; +using TagLib.Ape; + +namespace MediaBrowser.Providers.Trickplay; + +/// <summary> +/// Class TrickplayImagesTask. +/// </summary> +public class TrickplayImagesTask : IScheduledTask +{ + private const int QueryPageLimit = 100; + + private readonly ILogger<TrickplayImagesTask> _logger; + private readonly ILibraryManager _libraryManager; + private readonly ILocalizationManager _localization; + private readonly ITrickplayManager _trickplayManager; + + /// <summary> + /// Initializes a new instance of the <see cref="TrickplayImagesTask"/> class. + /// </summary> + /// <param name="logger">The logger.</param> + /// <param name="libraryManager">The library manager.</param> + /// <param name="localization">The localization manager.</param> + /// <param name="trickplayManager">The trickplay manager.</param> + public TrickplayImagesTask( + ILogger<TrickplayImagesTask> logger, + ILibraryManager libraryManager, + ILocalizationManager localization, + ITrickplayManager trickplayManager) + { + _libraryManager = libraryManager; + _logger = logger; + _localization = localization; + _trickplayManager = trickplayManager; + } + + /// <inheritdoc /> + public string Name => _localization.GetLocalizedString("TaskRefreshTrickplayImages"); + + /// <inheritdoc /> + public string Description => _localization.GetLocalizedString("TaskRefreshTrickplayImagesDescription"); + + /// <inheritdoc /> + public string Key => "RefreshTrickplayImages"; + + /// <inheritdoc /> + public string Category => _localization.GetLocalizedString("TasksLibraryCategory"); + + /// <inheritdoc /> + public IEnumerable<TaskTriggerInfo> GetDefaultTriggers() + { + return new[] + { + new TaskTriggerInfo + { + Type = TaskTriggerInfo.TriggerDaily, + TimeOfDayTicks = TimeSpan.FromHours(3).Ticks + } + }; + } + + /// <inheritdoc /> + public async Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken) + { + var query = new InternalItemsQuery + { + MediaTypes = new[] { MediaType.Video }, + SourceTypes = new[] { SourceType.Library }, + IsVirtualItem = false, + IsFolder = false, + Recursive = true, + Limit = QueryPageLimit + }; + + var numberOfVideos = _libraryManager.GetCount(query); + + var startIndex = 0; + var numComplete = 0; + + while (startIndex < numberOfVideos) + { + query.StartIndex = startIndex; + var videos = _libraryManager.GetItemList(query).OfType<Video>(); + + foreach (var video in videos) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await _trickplayManager.RefreshTrickplayDataAsync(video, false, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error creating trickplay files for {ItemName}", video.Name); + } + + numComplete++; + progress.Report(100d * numComplete / numberOfVideos); + } + + startIndex += QueryPageLimit; + } + + progress.Report(100); + } +} diff --git a/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs b/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs new file mode 100644 index 000000000..f6dcde4f6 --- /dev/null +++ b/MediaBrowser.Providers/Trickplay/TrickplayProvider.cs @@ -0,0 +1,121 @@ +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.Movies; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Controller.Trickplay; +using MediaBrowser.Model.Configuration; + +namespace MediaBrowser.Providers.Trickplay; + +/// <summary> +/// Class TrickplayProvider. Provides images and metadata for trickplay +/// scrubbing previews. +/// </summary> +public class TrickplayProvider : ICustomMetadataProvider<Episode>, + ICustomMetadataProvider<MusicVideo>, + ICustomMetadataProvider<Movie>, + ICustomMetadataProvider<Trailer>, + ICustomMetadataProvider<Video>, + IHasItemChangeMonitor, + IHasOrder, + IForcedProvider +{ + private readonly IServerConfigurationManager _config; + private readonly ITrickplayManager _trickplayManager; + private readonly ILibraryManager _libraryManager; + + /// <summary> + /// Initializes a new instance of the <see cref="TrickplayProvider"/> class. + /// </summary> + /// <param name="config">The configuration manager.</param> + /// <param name="trickplayManager">The trickplay manager.</param> + /// <param name="libraryManager">The library manager.</param> + public TrickplayProvider( + IServerConfigurationManager config, + ITrickplayManager trickplayManager, + ILibraryManager libraryManager) + { + _config = config; + _trickplayManager = trickplayManager; + _libraryManager = libraryManager; + } + + /// <inheritdoc /> + public string Name => "Trickplay Provider"; + + /// <inheritdoc /> + public int Order => 100; + + /// <inheritdoc /> + public bool HasChanged(BaseItem item, IDirectoryService directoryService) + { + if (item.IsFileProtocol) + { + var file = directoryService.GetFile(item.Path); + if (file is not null && item.DateModified != file.LastWriteTimeUtc) + { + return true; + } + } + + return false; + } + + /// <inheritdoc /> + public Task<ItemUpdateType> FetchAsync(Episode item, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + return FetchInternal(item, options, cancellationToken); + } + + /// <inheritdoc /> + public Task<ItemUpdateType> FetchAsync(MusicVideo item, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + return FetchInternal(item, options, cancellationToken); + } + + /// <inheritdoc /> + public Task<ItemUpdateType> FetchAsync(Movie item, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + return FetchInternal(item, options, cancellationToken); + } + + /// <inheritdoc /> + public Task<ItemUpdateType> FetchAsync(Trailer item, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + return FetchInternal(item, options, cancellationToken); + } + + /// <inheritdoc /> + public Task<ItemUpdateType> FetchAsync(Video item, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + return FetchInternal(item, options, cancellationToken); + } + + private async Task<ItemUpdateType> FetchInternal(Video video, MetadataRefreshOptions options, CancellationToken cancellationToken) + { + var libraryOptions = _libraryManager.GetLibraryOptions(video); + bool? enableDuringScan = libraryOptions?.ExtractTrickplayImagesDuringLibraryScan; + bool replace = options.ReplaceAllImages; + + if (options.IsAutomated && !enableDuringScan.GetValueOrDefault(false)) + { + return ItemUpdateType.None; + } + + if (_config.Configuration.TrickplayOptions.ScanBehavior == TrickplayScanBehavior.Blocking) + { + await _trickplayManager.RefreshTrickplayDataAsync(video, replace, cancellationToken).ConfigureAwait(false); + } + else + { + _ = _trickplayManager.RefreshTrickplayDataAsync(video, replace, cancellationToken).ConfigureAwait(false); + } + + // The core doesn't need to trigger any save operations over this + return ItemUpdateType.None; + } +} diff --git a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 03f90da8e..554fbe941 100644 --- a/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/src/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -2,14 +2,18 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; +using System.Security.Cryptography.Xml; using BlurHashSharp.SkiaSharp; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.Drawing; +using MediaBrowser.Controller.LiveTv; using MediaBrowser.Model.Drawing; using Microsoft.Extensions.Logging; using SkiaSharp; +using static System.Net.Mime.MediaTypeNames; using SKSvg = SkiaSharp.Extended.Svg.SKSvg; namespace Jellyfin.Drawing.Skia; @@ -515,6 +519,81 @@ public class SkiaEncoder : IImageEncoder splashBuilder.GenerateSplash(posters, backdrops, outputPath); } + /// <inheritdoc /> + public int CreateTrickplayTile(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight) + { + var paths = options.InputPaths; + var tileWidth = options.Width; + var tileHeight = options.Height; + + if (paths.Count < 1) + { + throw new ArgumentException("InputPaths cannot be empty."); + } + else if (paths.Count > tileWidth * tileHeight) + { + throw new ArgumentException($"InputPaths contains more images than would fit on {tileWidth}x{tileHeight} grid."); + } + + // If no height provided, use height of first image. + if (!imgHeight.HasValue) + { + using var firstImg = Decode(paths[0], false, null, out _); + + if (firstImg is null) + { + throw new InvalidDataException("Could not decode image data."); + } + + if (firstImg.Width != imgWidth) + { + throw new InvalidOperationException("Image width does not match provided width."); + } + + imgHeight = firstImg.Height; + } + + // Make horizontal strips using every provided image. + using var tileGrid = new SKBitmap(imgWidth * tileWidth, imgHeight.Value * tileHeight); + using var canvas = new SKCanvas(tileGrid); + + var imgIndex = 0; + for (var y = 0; y < tileHeight; y++) + { + for (var x = 0; x < tileWidth; x++) + { + if (imgIndex >= paths.Count) + { + break; + } + + using var img = Decode(paths[imgIndex++], false, null, out _); + + if (img is null) + { + throw new InvalidDataException("Could not decode image data."); + } + + if (img.Width != imgWidth) + { + throw new InvalidOperationException("Image width does not match provided width."); + } + + if (img.Height != imgHeight) + { + throw new InvalidOperationException("Image height does not match first image height."); + } + + canvas.DrawBitmap(img, x * imgWidth, y * imgHeight.Value); + } + } + + using var outputStream = new SKFileWStream(options.OutputPath); + tileGrid.Encode(outputStream, SKEncodedImageFormat.Jpeg, quality); + + return imgHeight.Value; + } + private void DrawIndicator(SKCanvas canvas, int imageWidth, int imageHeight, ImageProcessingOptions options) { try diff --git a/src/Jellyfin.Drawing/NullImageEncoder.cs b/src/Jellyfin.Drawing/NullImageEncoder.cs index 171128bed..1495661c1 100644 --- a/src/Jellyfin.Drawing/NullImageEncoder.cs +++ b/src/Jellyfin.Drawing/NullImageEncoder.cs @@ -50,6 +50,12 @@ public class NullImageEncoder : IImageEncoder } /// <inheritdoc /> + public int CreateTrickplayTile(ImageCollageOptions options, int quality, int imgWidth, int? imgHeight) + { + throw new NotImplementedException(); + } + + /// <inheritdoc /> public string GetImageBlurHash(int xComp, int yComp, string path) { throw new NotImplementedException(); |