Merge branch 'master' into fix/livetv-channel-icon-refresh

Resolve GuideManager conflict by keeping LiveTvChannelImageHelper so
channel icons re-fetch on every guide refresh, including when the URL
is unchanged.

1 files changed, 157 insertions, 59 deletions

diff --git a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
index e0c5f3ad39..67e323177b 100644
--- a/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
+++ b/MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs
@@ -26,7 +26,10 @@ using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.MediaInfo;
 using Microsoft.Extensions.Logging;
+using Nikse.SubtitleEdit.Core.Common;
+using Nikse.SubtitleEdit.Core.SubtitleFormats;
 using UtfUnknown;
+using SubtitleFormat = MediaBrowser.Model.MediaInfo.SubtitleFormat;
 
 namespace MediaBrowser.MediaEncoding.Subtitles
 {
@@ -72,55 +75,42 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
         private MemoryStream ConvertSubtitles(
             Stream stream,
-            string inputFormat,
+            SubtitleInfo inputInfo,
             string outputFormat,
             long startTimeTicks,
             long endTimeTicks,
-            bool preserveOriginalTimestamps,
-            CancellationToken cancellationToken)
+            bool preserveOriginalTimestamps)
         {
-            var ms = new MemoryStream();
+            var subtitle = Subtitle.Parse(stream, Path.GetExtension(inputInfo.Path));
 
-            try
-            {
-                var trackInfo = _subtitleParser.Parse(stream, inputFormat);
+            FilterEvents(subtitle, startTimeTicks, endTimeTicks, preserveOriginalTimestamps);
 
-                FilterEvents(trackInfo, startTimeTicks, endTimeTicks, preserveOriginalTimestamps);
+            var formatter = GetWriter(outputFormat);
 
-                var writer = GetWriter(outputFormat);
+            var text = formatter.ToText(subtitle, "untitled");
+            var bytes = Encoding.UTF8.GetBytes(text);
 
-                writer.Write(trackInfo, ms, cancellationToken);
-                ms.Position = 0;
-            }
-            catch
-            {
-                ms.Dispose();
-                throw;
-            }
-
-            return ms;
+            return new MemoryStream(bytes, 0, bytes.Length, false, true);
         }
 
-        internal void FilterEvents(SubtitleTrackInfo track, long startPositionTicks, long endTimeTicks, bool preserveTimestamps)
+        internal void FilterEvents(Subtitle track, long startPositionTicks, long endTimeTicks, bool preserveTimestamps)
         {
             // Drop subs that have fully elapsed before the requested start position
-            track.TrackEvents = track.TrackEvents
-                .SkipWhile(i => (i.StartPositionTicks - startPositionTicks) < 0 && (i.EndPositionTicks - startPositionTicks) < 0)
-                .ToArray();
+            track.Paragraphs
+                .RemoveAll(i => (i.StartTime.TimeSpan.Ticks - startPositionTicks) < 0 && (i.EndTime.TimeSpan.Ticks - startPositionTicks) < 0);
 
             if (endTimeTicks > 0)
             {
-                track.TrackEvents = track.TrackEvents
-                    .TakeWhile(i => i.StartPositionTicks <= endTimeTicks)
-                    .ToArray();
+                track.Paragraphs
+                    .RemoveAll(i => i.StartTime.TimeSpan.Ticks > endTimeTicks);
             }
 
             if (!preserveTimestamps)
             {
-                foreach (var trackEvent in track.TrackEvents)
+                foreach (var trackEvent in track.Paragraphs)
                 {
-                    trackEvent.EndPositionTicks = Math.Max(0, trackEvent.EndPositionTicks - startPositionTicks);
-                    trackEvent.StartPositionTicks = Math.Max(0, trackEvent.StartPositionTicks - startPositionTicks);
+                    trackEvent.StartTime = new TimeCode(TimeSpan.FromTicks(Math.Max(0, trackEvent.StartTime.TimeSpan.Ticks - startPositionTicks)));
+                    trackEvent.EndTime = new TimeCode(TimeSpan.FromTicks(Math.Max(0, trackEvent.EndTime.TimeSpan.Ticks - startPositionTicks)));
                 }
             }
         }
@@ -142,14 +132,14 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             var subtitleStream = mediaSource.MediaStreams
                .First(i => i.Type == MediaStreamType.Subtitle && i.Index == subtitleStreamIndex);
 
-            var (stream, inputFormat) = await GetSubtitleStream(mediaSource, subtitleStream, cancellationToken)
+            var (stream, info) = await GetSubtitleStream(mediaSource, subtitleStream, cancellationToken)
                         .ConfigureAwait(false);
 
             // Return the original if the same format is being requested
             // Character encoding was already handled in GetSubtitleStream
             // ASS is a superset of SSA, skipping the conversion and preserving the styles
-            if (string.Equals(inputFormat, outputFormat, StringComparison.OrdinalIgnoreCase)
-                || (string.Equals(inputFormat, SubtitleFormat.SSA, StringComparison.OrdinalIgnoreCase)
+            if (string.Equals(info.Format, outputFormat, StringComparison.OrdinalIgnoreCase)
+                || (string.Equals(info.Format, SubtitleFormat.SSA, StringComparison.OrdinalIgnoreCase)
                     && string.Equals(outputFormat, SubtitleFormat.ASS, StringComparison.OrdinalIgnoreCase)))
             {
                 return stream;
@@ -157,11 +147,11 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
             using (stream)
             {
-                return ConvertSubtitles(stream, inputFormat, outputFormat, startTimeTicks, endTimeTicks, preserveOriginalTimestamps, cancellationToken);
+                return ConvertSubtitles(stream, info, outputFormat, startTimeTicks, endTimeTicks, preserveOriginalTimestamps);
             }
         }
 
-        private async Task<(Stream Stream, string Format)> GetSubtitleStream(
+        private async Task<(Stream Stream, SubtitleInfo Info)> GetSubtitleStream(
             MediaSourceInfo mediaSource,
             MediaStream subtitleStream,
             CancellationToken cancellationToken)
@@ -170,7 +160,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
             var stream = await GetSubtitleStream(fileInfo, cancellationToken).ConfigureAwait(false);
 
-            return (stream, fileInfo.Format);
+            return (stream, fileInfo);
         }
 
         private async Task<Stream> GetSubtitleStream(SubtitleInfo fileInfo, CancellationToken cancellationToken)
@@ -220,12 +210,12 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     Path = outputPath,
                     Protocol = MediaProtocol.File,
                     Format = outputFormat,
-                    IsExternal = false
+                    IsExternal = MediaStream.IsVobSubFormat(outputFormat)
                 };
             }
 
-            var currentFormat = subtitleStream.Codec ?? Path.GetExtension(subtitleStream.Path)
-                .TrimStart('.');
+            // Normalize ffmpeg codec names to the file extensions the parser is keyed on
+            var currentFormat = NormalizeCodecToParserExtension((Path.GetExtension(subtitleStream.Path) ?? subtitleStream.Codec).TrimStart('.'));
 
             // Handle PGS subtitles as raw streams for the client to render
             if (MediaStream.IsPgsFormat(currentFormat))
@@ -267,13 +257,13 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             };
         }
 
-        private bool TryGetWriter(string format, [NotNullWhen(true)] out ISubtitleWriter? value)
+        private bool TryGetWriter(string format, [NotNullWhen(true)] out Nikse.SubtitleEdit.Core.SubtitleFormats.SubtitleFormat? value)
         {
             ArgumentException.ThrowIfNullOrEmpty(format);
 
             if (string.Equals(format, SubtitleFormat.ASS, StringComparison.OrdinalIgnoreCase))
             {
-                value = new AssWriter();
+                value = new AdvancedSubStationAlpha();
                 return true;
             }
 
@@ -283,27 +273,29 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 return true;
             }
 
-            if (string.Equals(format, SubtitleFormat.SRT, StringComparison.OrdinalIgnoreCase) || string.Equals(format, SubtitleFormat.SUBRIP, StringComparison.OrdinalIgnoreCase))
+            if (string.Equals(format, SubtitleFormat.SRT, StringComparison.OrdinalIgnoreCase)
+                || string.Equals(format, SubtitleFormat.SUBRIP, StringComparison.OrdinalIgnoreCase))
             {
-                value = new SrtWriter();
+                value = new SubRip();
                 return true;
             }
 
             if (string.Equals(format, SubtitleFormat.SSA, StringComparison.OrdinalIgnoreCase))
             {
-                value = new SsaWriter();
+                value = new SubStationAlpha();
                 return true;
             }
 
-            if (string.Equals(format, SubtitleFormat.VTT, StringComparison.OrdinalIgnoreCase) || string.Equals(format, SubtitleFormat.WEBVTT, StringComparison.OrdinalIgnoreCase))
+            if (string.Equals(format, SubtitleFormat.VTT, StringComparison.OrdinalIgnoreCase)
+                || string.Equals(format, SubtitleFormat.WEBVTT, StringComparison.OrdinalIgnoreCase))
             {
-                value = new VttWriter();
+                value = new WebVTT();
                 return true;
             }
 
             if (string.Equals(format, SubtitleFormat.TTML, StringComparison.OrdinalIgnoreCase))
             {
-                value = new TtmlWriter();
+                value = new TimedText10();
                 return true;
             }
 
@@ -311,7 +303,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             return false;
         }
 
-        private ISubtitleWriter GetWriter(string format)
+        private Nikse.SubtitleEdit.Core.SubtitleFormats.SubtitleFormat GetWriter(string format)
         {
             if (TryGetWriter(format, out var writer))
             {
@@ -333,13 +325,91 @@ namespace MediaBrowser.MediaEncoding.Subtitles
         {
             using (await _semaphoreLocks.LockAsync(outputPath, cancellationToken).ConfigureAwait(false))
             {
-                if (!File.Exists(outputPath) || _fileSystem.GetFileInfo(outputPath).Length == 0)
+                if (!IsCachedSubtitleFresh(outputPath, subtitleStream.Path))
                 {
                     await ConvertTextSubtitleToSrtInternal(subtitleStream, mediaSource, outputPath, cancellationToken).ConfigureAwait(false);
                 }
             }
         }
 
+        // ffmpeg codec names don't always match the file extensions the subtitle parser is keyed on.
+        private static string NormalizeCodecToParserExtension(string codecOrExtension)
+        {
+            return codecOrExtension switch
+            {
+                "subrip" => "srt",
+                "webvtt" => "vtt",
+                _ => codecOrExtension
+            };
+        }
+
+        // Records "this cache was built from this exact source revision" in a sidecar file next to the cache: "<sizeBytes>:<mtimeTicks>"
+        private static string GetCacheMetaPath(string cachePath) => cachePath + ".meta";
+
+        private static string FormatCacheMeta(long length, DateTime lastWriteUtc)
+            => string.Create(CultureInfo.InvariantCulture, $"{length}:{lastWriteUtc.Ticks}");
+
+        private bool IsCachedSubtitleFresh(string cachePath, string? sourcePath)
+        {
+            if (!File.Exists(cachePath))
+            {
+                return false;
+            }
+
+            var cacheInfo = _fileSystem.GetFileInfo(cachePath);
+            if (cacheInfo.Length == 0)
+            {
+                return false;
+            }
+
+            if (string.IsNullOrEmpty(sourcePath) || !File.Exists(sourcePath))
+            {
+                return true;
+            }
+
+            var metaPath = GetCacheMetaPath(cachePath);
+            if (!File.Exists(metaPath))
+            {
+                // Pre-existing cache from before metadata tracking - regenerate so we can record the source state.
+                return false;
+            }
+
+            try
+            {
+                var sourceInfo = _fileSystem.GetFileInfo(sourcePath);
+                var expected = FormatCacheMeta(sourceInfo.Length, sourceInfo.LastWriteTimeUtc);
+                var actual = File.ReadAllText(metaPath);
+                return string.Equals(expected, actual, StringComparison.Ordinal);
+            }
+            catch (IOException)
+            {
+                return false;
+            }
+        }
+
+        private void WriteCacheMeta(string cachePath, string? sourcePath)
+        {
+            if (string.IsNullOrEmpty(sourcePath))
+            {
+                return;
+            }
+
+            try
+            {
+                var sourceInfo = _fileSystem.GetFileInfo(sourcePath);
+                if (!sourceInfo.Exists)
+                {
+                    return;
+                }
+
+                File.WriteAllText(GetCacheMetaPath(cachePath), FormatCacheMeta(sourceInfo.Length, sourceInfo.LastWriteTimeUtc));
+            }
+            catch (IOException ex)
+            {
+                _logger.LogWarning(ex, "Failed to record subtitle cache metadata for {CachePath}", cachePath);
+            }
+        }
+
         /// <summary>
         /// Converts the text subtitle to SRT internal.
         /// </summary>
@@ -384,7 +454,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     CreateNoWindow = true,
                     UseShellExecute = false,
                     FileName = _mediaEncoder.EncoderPath,
-                    Arguments = string.Format(CultureInfo.InvariantCulture, "{0} -i \"{1}\" -c:s srt \"{2}\"", encodingParam, inputPath, outputPath),
+                    Arguments = string.Format(CultureInfo.InvariantCulture, "-y {0} -i \"{1}\" -c:s srt \"{2}\"", encodingParam, inputPath, outputPath),
                     WindowStyle = ProcessWindowStyle.Hidden,
                     ErrorDialog = false
                 },
@@ -464,6 +534,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
             await SetAssFont(outputPath, cancellationToken).ConfigureAwait(false);
 
+            WriteCacheMeta(outputPath, inputPath);
+
             _logger.LogInformation("ffmpeg subtitle conversion succeeded for {Path}", inputPath);
         }
 
@@ -475,6 +547,10 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             {
                 return subtitleStream.Codec;
             }
+            else if (MediaStream.IsVobSubFormat(subtitleStream.Codec))
+            {
+                return "mks";
+            }
             else
             {
                 return "srt";
@@ -488,6 +564,11 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             {
                 return "sup";
             }
+            else if (MediaStream.IsVobSubFormat(subtitleStream.Codec))
+            {
+                // FFmpeg cannot mux VobSub subtitle streams back into the .idx/.sub pair, so we use .mks container instead.
+                return "mks";
+            }
             else
             {
                 return GetExtractableSubtitleFormat(subtitleStream);
@@ -500,7 +581,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 || string.Equals(codec, "ssa", StringComparison.OrdinalIgnoreCase)
                 || string.Equals(codec, "srt", StringComparison.OrdinalIgnoreCase)
                 || string.Equals(codec, "subrip", StringComparison.OrdinalIgnoreCase)
-                || string.Equals(codec, "pgssub", StringComparison.OrdinalIgnoreCase);
+                || string.Equals(codec, "pgssub", StringComparison.OrdinalIgnoreCase)
+                || MediaStream.IsVobSubFormat(codec);
         }
 
         /// <inheritdoc />
@@ -516,7 +598,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
                 foreach (var subtitleStream in subtitleStreams)
                 {
-                    if (subtitleStream.IsExternal && !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
+                    if (subtitleStream.IsExternal
+                        && !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
                     {
                         continue;
                     }
@@ -529,7 +612,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
 
                     var releaser = await _semaphoreLocks.LockAsync(outputPath, cancellationToken).ConfigureAwait(false);
 
-                    if (File.Exists(outputPath) && _fileSystem.GetFileInfo(outputPath).Length > 0)
+                    var sourcePath = string.IsNullOrEmpty(subtitleStream.Path) ? mediaSource.Path : subtitleStream.Path;
+                    if (IsCachedSubtitleFresh(outputPath, sourcePath))
                     {
                         releaser.Dispose();
                         continue;
@@ -586,7 +670,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 var outputPaths = new List<string>();
                 var args = string.Format(
                     CultureInfo.InvariantCulture,
-                    "-i {0}",
+                    "-y -i {0}",
                     inputPath);
 
                 foreach (var subtitleStream in subtitleStreams)
@@ -603,6 +687,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     }
 
                     var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
+                    // FFmpeg does not provide an .idx/.sub muxer, so VobSub streams must be written as MKS files.
+                    var outputFormatOption = MediaStream.IsVobSubFormat(subtitleStream.Codec) ? " -f matroska" : string.Empty;
                     var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
 
                     if (streamIndex == -1)
@@ -616,13 +702,19 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                     outputPaths.Add(outputPath);
                     args += string.Format(
                         CultureInfo.InvariantCulture,
-                        " -map 0:{0} -an -vn -c:s {1} -flush_packets 1 \"{2}\"",
+                        " -map 0:{0} -an -vn -c:s {1}{2} -flush_packets 1 \"{3}\"",
                         streamIndex,
                         outputCodec,
+                        outputFormatOption,
                         outputPath);
                 }
 
                 await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
+
+                foreach (var outputPath in outputPaths)
+                {
+                    WriteCacheMeta(outputPath, mksFile);
+                }
             }
         }
 
@@ -653,6 +745,8 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 }
 
                 var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
+                // FFmpeg does not provide an .idx/.sub muxer, so VobSub streams must be written as MKS files.
+                var outputFormatOption = MediaStream.IsVobSubFormat(subtitleStream.Codec) ? " -f matroska" : string.Empty;
                 var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
 
                 if (streamIndex == -1)
@@ -666,18 +760,22 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 outputPaths.Add(outputPath);
                 args += string.Format(
                     CultureInfo.InvariantCulture,
-                    " -map 0:{0} -an -vn -c:s {1} -flush_packets 1 \"{2}\"",
+                    " -map 0:{0} -an -vn -c:s {1}{2} -flush_packets 1 \"{3}\"",
                     streamIndex,
                     outputCodec,
+                    outputFormatOption,
                     outputPath);
             }
 
-            if (outputPaths.Count == 0)
+            if (outputPaths.Count > 0)
             {
-                return;
-            }
+                await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
 
-            await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
+                foreach (var outputPath in outputPaths)
+                {
+                    WriteCacheMeta(outputPath, mediaSource.Path);
+                }
+            }
         }
 
         private async Task ExtractSubtitlesForFile(