From eca9bf41bcf536708ad74236793b363db3af1e4d Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sat, 16 Mar 2024 03:40:14 +0800 Subject: Add TranscodingSegmentCleaner to replace ffmpeg's hlsenc deletion FFmpeg deletes segments based on its own transcoding progress, but we need to delete segments based on client download progress. Since disk and GPU speeds vary, using hlsenc's built-in deletion will result in premature deletion of some segments. As a consequence, the server has to constantly respin new ffmpeg instances, resulting in choppy video playback. Signed-off-by: nyanmisaka --- .../MediaEncoding/TranscodingSegmentCleaner.cs | 188 +++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs (limited to 'MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs') diff --git a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs new file mode 100644 index 000000000..6cbda8e0a --- /dev/null +++ b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Model.Configuration; +using MediaBrowser.Model.IO; +using Microsoft.Extensions.Logging; + +namespace MediaBrowser.Controller.MediaEncoding; + +/// +/// Transcoding segment cleaner. +/// +public class TranscodingSegmentCleaner : IDisposable +{ + private readonly TranscodingJob _job; + private readonly ILogger _logger; + private readonly IConfigurationManager _config; + private readonly IFileSystem _fileSystem; + private readonly IMediaEncoder _mediaEncoder; + private Timer? _timer; + private int _segmentLength; + + /// + /// Initializes a new instance of the class. + /// + /// Transcoding job dto. + /// Instance of the interface. + /// Instance of the interface. + /// Instance of the interface. + /// Instance of the interface. + /// The segment length of this transcoding job. + public TranscodingSegmentCleaner(TranscodingJob job, ILogger logger, IConfigurationManager config, IFileSystem fileSystem, IMediaEncoder mediaEncoder, int segmentLength) + { + _job = job; + _logger = logger; + _config = config; + _fileSystem = fileSystem; + _mediaEncoder = mediaEncoder; + _segmentLength = segmentLength; + } + + /// + /// Start timer. + /// + public void Start() + { + _timer = new Timer(TimerCallback, null, 20000, 20000); + } + + /// + /// Stop cleaner. + /// + public void Stop() + { + DisposeTimer(); + } + + /// + /// Dispose cleaner. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Dispose cleaner. + /// + /// Disposing. + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + DisposeTimer(); + } + } + + private EncodingOptions GetOptions() + { + return _config.GetEncodingOptions(); + } + + private async void TimerCallback(object? state) + { + if (_job.HasExited) + { + DisposeTimer(); + return; + } + + var options = GetOptions(); + var enableSegmentDeletion = options.EnableSegmentDeletion; + var segmentKeepSeconds = Math.Max(options.SegmentKeepSeconds, 20); + + if (enableSegmentDeletion) + { + var downloadPositionTicks = _job.DownloadPositionTicks ?? 0; + var downloadPositionSeconds = Convert.ToInt64(TimeSpan.FromTicks(downloadPositionTicks).TotalSeconds); + + if (downloadPositionSeconds > 0 && segmentKeepSeconds > 0 && downloadPositionSeconds > segmentKeepSeconds) + { + var idxMaxToRemove = (downloadPositionSeconds - segmentKeepSeconds) / _segmentLength; + + if (idxMaxToRemove > 0) + { + await DeleteSegmentFiles(_job, 0, idxMaxToRemove, 0, 1500).ConfigureAwait(false); + } + } + } + } + + private async Task DeleteSegmentFiles(TranscodingJob job, long idxMin, long idxMax, int retryCount, int delayMs) + { + if (retryCount >= 10) + { + return; + } + + var path = job.Path ?? throw new ArgumentException("Path can't be null."); + + _logger.LogDebug("Deleting segment file(s) index {Min} to {Max} from {Path}", idxMin, idxMax, path); + + await Task.Delay(delayMs).ConfigureAwait(false); + + try + { + if (job.Type == TranscodingJobType.Hls) + { + DeleteHlsSegmentFiles(path, idxMin, idxMax); + } + } + catch (IOException ex) + { + _logger.LogError(ex, "Error deleting segment file(s) {Path}", path); + + await DeleteSegmentFiles(job, idxMin, idxMax, retryCount + 1, 500).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error deleting segment file(s) {Path}", path); + } + } + + private void DeleteHlsSegmentFiles(string outputFilePath, long idxMin, long idxMax) + { + var directory = Path.GetDirectoryName(outputFilePath) + ?? throw new ArgumentException("Path can't be a root directory.", nameof(outputFilePath)); + + var name = Path.GetFileNameWithoutExtension(outputFilePath); + + var filesToDelete = _fileSystem.GetFilePaths(directory) + .Where(f => long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx) && idx >= idxMin && idx <= idxMax); + + List? exs = null; + foreach (var file in filesToDelete) + { + try + { + _logger.LogDebug("Deleting HLS segment file {0}", file); + _fileSystem.DeleteFile(file); + } + catch (IOException ex) + { + (exs ??= new List(4)).Add(ex); + _logger.LogError(ex, "Error deleting HLS segment file {Path}", file); + } + } + + if (exs is not null) + { + throw new AggregateException("Error deleting HLS segment files", exs); + } + } + + private void DisposeTimer() + { + if (_timer is not null) + { + _timer.Dispose(); + _timer = null; + } + } +} -- cgit v1.2.3 From 50541aea91629f11b1ead72b059f09c91dd758ab Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sat, 16 Mar 2024 22:09:31 +0800 Subject: Apply suggestions from code review Add excludeFilePaths to skip segment files in which IOException occurred. Signed-off-by: nyanmisaka --- .../MediaEncoding/TranscodingSegmentCleaner.cs | 26 +++++++++------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs') diff --git a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs index 6cbda8e0a..d18f26b8b 100644 --- a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs +++ b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs @@ -23,6 +23,7 @@ public class TranscodingSegmentCleaner : IDisposable private readonly IMediaEncoder _mediaEncoder; private Timer? _timer; private int _segmentLength; + private List? _excludeFilePaths; /// /// Initializes a new instance of the class. @@ -41,6 +42,7 @@ public class TranscodingSegmentCleaner : IDisposable _fileSystem = fileSystem; _mediaEncoder = mediaEncoder; _segmentLength = segmentLength; + _excludeFilePaths = null; } /// @@ -104,23 +106,18 @@ public class TranscodingSegmentCleaner : IDisposable if (downloadPositionSeconds > 0 && segmentKeepSeconds > 0 && downloadPositionSeconds > segmentKeepSeconds) { - var idxMaxToRemove = (downloadPositionSeconds - segmentKeepSeconds) / _segmentLength; + var idxMaxToDelete = (downloadPositionSeconds - segmentKeepSeconds) / _segmentLength; - if (idxMaxToRemove > 0) + if (idxMaxToDelete > 0) { - await DeleteSegmentFiles(_job, 0, idxMaxToRemove, 0, 1500).ConfigureAwait(false); + await DeleteSegmentFiles(_job, 0, idxMaxToDelete, 1500).ConfigureAwait(false); } } } } - private async Task DeleteSegmentFiles(TranscodingJob job, long idxMin, long idxMax, int retryCount, int delayMs) + private async Task DeleteSegmentFiles(TranscodingJob job, long idxMin, long idxMax, int delayMs) { - if (retryCount >= 10) - { - return; - } - var path = job.Path ?? throw new ArgumentException("Path can't be null."); _logger.LogDebug("Deleting segment file(s) index {Min} to {Max} from {Path}", idxMin, idxMax, path); @@ -134,12 +131,6 @@ public class TranscodingSegmentCleaner : IDisposable DeleteHlsSegmentFiles(path, idxMin, idxMax); } } - catch (IOException ex) - { - _logger.LogError(ex, "Error deleting segment file(s) {Path}", path); - - await DeleteSegmentFiles(job, idxMin, idxMax, retryCount + 1, 500).ConfigureAwait(false); - } catch (Exception ex) { _logger.LogError(ex, "Error deleting segment file(s) {Path}", path); @@ -154,7 +145,9 @@ public class TranscodingSegmentCleaner : IDisposable var name = Path.GetFileNameWithoutExtension(outputFilePath); var filesToDelete = _fileSystem.GetFilePaths(directory) - .Where(f => long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx) && idx >= idxMin && idx <= idxMax); + .Where(f => (!_excludeFilePaths?.Contains(f) ?? true) + && long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx) + && (idx >= idxMin && idx <= idxMax)); List? exs = null; foreach (var file in filesToDelete) @@ -167,6 +160,7 @@ public class TranscodingSegmentCleaner : IDisposable catch (IOException ex) { (exs ??= new List(4)).Add(ex); + (_excludeFilePaths ??= new List()).Add(file); _logger.LogError(ex, "Error deleting HLS segment file {Path}", file); } } -- cgit v1.2.3 From 47a77974b8a85bff007f439d9c9291220069017a Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sun, 17 Mar 2024 17:17:47 +0800 Subject: Apply suggestions from code review Drop excludeFilePaths and lower the log level to debug to avoid spamming in the log file. Signed-off-by: nyanmisaka --- .../MediaEncoding/TranscodingSegmentCleaner.cs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs') diff --git a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs index d18f26b8b..a6d812873 100644 --- a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs +++ b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs @@ -23,7 +23,6 @@ public class TranscodingSegmentCleaner : IDisposable private readonly IMediaEncoder _mediaEncoder; private Timer? _timer; private int _segmentLength; - private List? _excludeFilePaths; /// /// Initializes a new instance of the class. @@ -42,7 +41,6 @@ public class TranscodingSegmentCleaner : IDisposable _fileSystem = fileSystem; _mediaEncoder = mediaEncoder; _segmentLength = segmentLength; - _excludeFilePaths = null; } /// @@ -133,7 +131,7 @@ public class TranscodingSegmentCleaner : IDisposable } catch (Exception ex) { - _logger.LogError(ex, "Error deleting segment file(s) {Path}", path); + _logger.LogDebug(ex, "Error deleting segment file(s) {Path}", path); } } @@ -145,8 +143,7 @@ public class TranscodingSegmentCleaner : IDisposable var name = Path.GetFileNameWithoutExtension(outputFilePath); var filesToDelete = _fileSystem.GetFilePaths(directory) - .Where(f => (!_excludeFilePaths?.Contains(f) ?? true) - && long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx) + .Where(f => long.TryParse(Path.GetFileNameWithoutExtension(f).Replace(name, string.Empty, StringComparison.Ordinal), out var idx) && (idx >= idxMin && idx <= idxMax)); List? exs = null; @@ -160,8 +157,7 @@ public class TranscodingSegmentCleaner : IDisposable catch (IOException ex) { (exs ??= new List(4)).Add(ex); - (_excludeFilePaths ??= new List()).Add(file); - _logger.LogError(ex, "Error deleting HLS segment file {Path}", file); + _logger.LogDebug(ex, "Error deleting HLS segment file {Path}", file); } } -- cgit v1.2.3 From 557b8f0c7879261d2dc8268e77836fd6efb7feb8 Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sun, 17 Mar 2024 20:45:00 +0800 Subject: Apply suggestions from code review Drop the unnecessary initial capacity from the list. Signed-off-by: nyanmisaka --- MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs | 2 +- MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs') diff --git a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs index a6d812873..67bfcb02f 100644 --- a/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs +++ b/MediaBrowser.Controller/MediaEncoding/TranscodingSegmentCleaner.cs @@ -156,7 +156,7 @@ public class TranscodingSegmentCleaner : IDisposable } catch (IOException ex) { - (exs ??= new List(4)).Add(ex); + (exs ??= new List()).Add(ex); _logger.LogDebug(ex, "Error deleting HLS segment file {Path}", file); } } diff --git a/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs b/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs index 2a72cacdc..499e5287a 100644 --- a/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs +++ b/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs @@ -321,7 +321,7 @@ public sealed class TranscodeManager : ITranscodeManager, IDisposable } catch (IOException ex) { - (exs ??= new List(4)).Add(ex); + (exs ??= new List()).Add(ex); _logger.LogError(ex, "Error deleting HLS file {Path}", file); } } -- cgit v1.2.3