Enable VAAPI decoding without hardware encoding

Enable VAAPI command arguments to ffmpeg if VAAPI is selected, and
add the "hwdownload" filter if transcoding from VAAPI to software.
Also support transforming 10 bit colourspace to 8-bit, consistent
with other hardware encoding options, at least until client pixel
formats are configurable.

1 files changed, 17 insertions, 3 deletions

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index 87874001a..6b83edfff 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -459,7 +459,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             if (state.IsVideoRequest)
             {
-                if (GetVideoEncoder(state, encodingOptions).IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1)
+                if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
                 {
                     var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
                     var hwOutputFormat = "vaapi";
@@ -1780,8 +1780,24 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             var request = state.BaseRequest;
 
+            var videoStream = state.VideoStream;
             var filters = new List<string>();
 
+            // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
+            var hwType = options.HardwareAccelerationType ?? string.Empty;
+            if (string.Equals(hwType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding )
+            {
+                filters.Add("hwdownload");
+
+                // If transcoding from 10 bit, transform colour spaces too
+                if ( !string.IsNullOrEmpty(videoStream.PixelFormat) && videoStream.PixelFormat.IndexOf( "p10", StringComparison.OrdinalIgnoreCase ) != -1
+                    && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase ) )
+                {
+                    filters.Add("format=p010le");
+                    filters.Add("format=nv12");
+                }
+            }
+
             if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
             {
                 filters.Add("format=nv12|vaapi");
@@ -1793,8 +1809,6 @@ namespace MediaBrowser.Controller.MediaEncoding
                 filters.Add(string.Format("deinterlace_vaapi"));
             }
 
-            var videoStream = state.VideoStream;
-
             if ((state.DeInterlace("h264", true) || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true)) &&
                 !string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
             {