Merge pull request #28 from MediaBrowser/dev

Dev

1 files changed, 755 insertions, 164 deletions

diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs
index 5ece3dd82..1656d8304 100644
--- a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs
+++ b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs
@@ -19,11 +19,15 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using MediaBrowser.Common.Extensions;
+using MediaBrowser.Common.IO;
 using MediaBrowser.Controller.Channels;
+using MediaBrowser.Controller.Collections;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Playlists;
 using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.LiveTv;
+using MediaBrowser.Server.Implementations.Devices;
+using MediaBrowser.Server.Implementations.Playlists;
 
 namespace MediaBrowser.Server.Implementations.Persistence
 {
@@ -95,11 +99,12 @@ namespace MediaBrowser.Server.Implementations.Persistence
         private IDbCommand _updateInheritedTagsCommand;
 
         public const int LatestSchemaVersion = 109;
+        private readonly IMemoryStreamProvider _memoryStreamProvider;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="SqliteItemRepository"/> class.
         /// </summary>
-        public SqliteItemRepository(IServerConfigurationManager config, IJsonSerializer jsonSerializer, ILogManager logManager, IDbConnector connector)
+        public SqliteItemRepository(IServerConfigurationManager config, IJsonSerializer jsonSerializer, ILogManager logManager, IDbConnector connector, IMemoryStreamProvider memoryStreamProvider)
             : base(logManager, connector)
         {
             if (config == null)
@@ -113,6 +118,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
             _config = config;
             _jsonSerializer = jsonSerializer;
+            _memoryStreamProvider = memoryStreamProvider;
 
             _criticReviewsPath = Path.Combine(_config.ApplicationPaths.DataPath, "critic-reviews");
             DbFilePath = Path.Combine(_config.ApplicationPaths.DataPath, "library.db");
@@ -125,7 +131,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
             var cacheSize = _config.Configuration.SqliteCacheSize;
             if (cacheSize <= 0)
             {
-                cacheSize = Math.Min(Environment.ProcessorCount * 50000, 200000);
+                cacheSize = Math.Min(Environment.ProcessorCount * 50000, 100000);
             }
 
             var connection = await DbConnector.Connect(DbFilePath, false, false, 0 - cacheSize).ConfigureAwait(false);
@@ -270,6 +276,15 @@ namespace MediaBrowser.Server.Implementations.Persistence
             _connection.AddColumn(Logger, "TypedBaseItems", "SeasonId", "GUID");
             _connection.AddColumn(Logger, "TypedBaseItems", "SeriesId", "GUID");
             _connection.AddColumn(Logger, "TypedBaseItems", "SeriesSortName", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ExternalSeriesId", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ShortOverview", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "Tagline", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "Keywords", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ProviderIds", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "Images", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ProductionLocations", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ThemeSongIds", "Text");
+            _connection.AddColumn(Logger, "TypedBaseItems", "ThemeVideoIds", "Text");
 
             _connection.AddColumn(Logger, "UserDataKeys", "Priority", "INT");
             _connection.AddColumn(Logger, "ItemValues", "CleanValue", "Text");
@@ -412,7 +427,17 @@ namespace MediaBrowser.Server.Implementations.Persistence
             "SeriesId",
             "SeriesSortName",
             "PresentationUniqueKey",
-            "InheritedParentalRatingValue"
+            "InheritedParentalRatingValue",
+            "InheritedTags",
+            "ExternalSeriesId",
+            "ShortOverview",
+            "Tagline",
+            "Keywords",
+            "ProviderIds",
+            "Images",
+            "ProductionLocations",
+            "ThemeSongIds",
+            "ThemeVideoIds"
         };
 
         private readonly string[] _mediaStreamSaveColumns =
@@ -534,7 +559,16 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 "SeasonName",
                 "SeasonId",
                 "SeriesId",
-                "SeriesSortName"
+                "SeriesSortName",
+                "ExternalSeriesId",
+                "ShortOverview",
+                "Tagline",
+                "Keywords",
+                "ProviderIds",
+                "Images",
+                "ProductionLocations",
+                "ThemeSongIds",
+                "ThemeVideoIds"
             };
             _saveItemCommand = _connection.CreateCommand();
             _saveItemCommand.CommandText = "replace into TypedBaseItems (" + string.Join(",", saveColumns.ToArray()) + ") values (";
@@ -720,7 +754,15 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
                     _saveItemCommand.GetParameter(index++).Value = item.Id;
                     _saveItemCommand.GetParameter(index++).Value = item.GetType().FullName;
-                    _saveItemCommand.GetParameter(index++).Value = _jsonSerializer.SerializeToBytes(item);
+
+                    if (TypeRequiresDeserialization(item.GetType()))
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = _jsonSerializer.SerializeToBytes(item, _memoryStreamProvider);
+                    }
+                    else
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = null;
+                    }
 
                     _saveItemCommand.GetParameter(index++).Value = item.Path;
 
@@ -974,6 +1016,49 @@ namespace MediaBrowser.Server.Implementations.Persistence
                         _saveItemCommand.GetParameter(index++).Value = null;
                     }
 
+                    _saveItemCommand.GetParameter(index++).Value = item.ExternalSeriesId;
+                    _saveItemCommand.GetParameter(index++).Value = item.ShortOverview;
+                    _saveItemCommand.GetParameter(index++).Value = item.Tagline;
+
+                    if (item.Keywords.Count > 0)
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = string.Join("|", item.Keywords.ToArray());
+                    }
+                    else
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = null;
+                    }
+
+                    _saveItemCommand.GetParameter(index++).Value = SerializeProviderIds(item);
+                    _saveItemCommand.GetParameter(index++).Value = SerializeImages(item);
+
+                    if (item.ProductionLocations.Count > 0)
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = string.Join("|", item.ProductionLocations.ToArray());
+                    }
+                    else
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = null;
+                    }
+
+                    if (item.ThemeSongIds.Count > 0)
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = string.Join("|", item.ThemeSongIds.Select(i => i.ToString("N")).ToArray());
+                    }
+                    else
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = null;
+                    }
+
+                    if (item.ThemeVideoIds.Count > 0)
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = string.Join("|", item.ThemeVideoIds.Select(i => i.ToString("N")).ToArray());
+                    }
+                    else
+                    {
+                        _saveItemCommand.GetParameter(index++).Value = null;
+                    }
+
                     _saveItemCommand.Transaction = transaction;
 
                     _saveItemCommand.ExecuteNonQuery();
@@ -1022,6 +1107,99 @@ namespace MediaBrowser.Server.Implementations.Persistence
             }
         }
 
+        private string SerializeProviderIds(BaseItem item)
+        {
+            var ids = item.ProviderIds.ToList();
+
+            if (ids.Count == 0)
+            {
+                return null;
+            }
+
+            return string.Join("|", ids.Select(i => i.Key + "=" + i.Value).ToArray());
+        }
+
+        private void DeserializeProviderIds(string value, BaseItem item)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                return;
+            }
+
+            if (item.ProviderIds.Count > 0)
+            {
+                return;
+            }
+
+            var parts = value.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
+
+            foreach (var part in parts)
+            {
+                var idParts = part.Split('=');
+
+                item.SetProviderId(idParts[0], idParts[1]);
+            }
+        }
+
+        private string SerializeImages(BaseItem item)
+        {
+            var images = item.ImageInfos.ToList();
+
+            if (images.Count == 0)
+            {
+                return null;
+            }
+
+            return string.Join("|", images.Select(ToValueString).ToArray());
+        }
+
+        private void DeserializeImages(string value, BaseItem item)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                return;
+            }
+
+            if (item.ImageInfos.Count > 0)
+            {
+                return;
+            }
+
+            var parts = value.Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
+
+            foreach (var part in parts)
+            {
+                item.ImageInfos.Add(ItemImageInfoFromValueString(part));
+            }
+        }
+
+        public string ToValueString(ItemImageInfo image)
+        {
+            var delimeter = "*";
+
+            return (image.Path ?? string.Empty) +
+                delimeter +
+                image.DateModified.Ticks.ToString(CultureInfo.InvariantCulture) +
+                delimeter +
+                image.Type +
+                delimeter +
+                image.IsPlaceholder;
+        }
+
+        public ItemImageInfo ItemImageInfoFromValueString(string value)
+        {
+            var parts = value.Split(new[] { '*' }, StringSplitOptions.RemoveEmptyEntries);
+
+            var image = new ItemImageInfo();
+
+            image.Path = parts[0];
+            image.DateModified = new DateTime(long.Parse(parts[1], CultureInfo.InvariantCulture), DateTimeKind.Utc);
+            image.Type = (ImageType)Enum.Parse(typeof(ImageType), parts[2], true);
+            image.IsPlaceholder = string.Equals(parts[3], true.ToString(), StringComparison.OrdinalIgnoreCase);
+
+            return image;
+        }
+
         /// <summary>
         /// Internal retrieve from items or users table
         /// </summary>
@@ -1056,6 +1234,91 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
         private BaseItem GetItem(IDataReader reader)
         {
+            return GetItem(reader, new InternalItemsQuery());
+        }
+
+        private bool TypeRequiresDeserialization(Type type)
+        {
+            if (_config.Configuration.SkipDeserializationForBasicTypes)
+            {
+                if (type == typeof(MusicGenre))
+                {
+                    return false;
+                }
+                if (type == typeof(GameGenre))
+                {
+                    return false;
+                }
+                if (type == typeof(Genre))
+                {
+                    return false;
+                }
+                if (type == typeof(Studio))
+                {
+                    return false;
+                }
+                if (type == typeof(Year))
+                {
+                    return false;
+                }
+                if (type == typeof(Book))
+                {
+                    return false;
+                }
+                if (type == typeof(Person))
+                {
+                    return false;
+                }
+                if (type == typeof(RecordingGroup))
+                {
+                    return false;
+                }
+                if (type == typeof(Channel))
+                {
+                    return false;
+                }
+                if (type == typeof(ManualCollectionsFolder))
+                {
+                    return false;
+                }
+                if (type == typeof(CameraUploadsFolder))
+                {
+                    return false;
+                }
+                if (type == typeof(PlaylistsFolder))
+                {
+                    return false;
+                }
+                if (type == typeof(UserRootFolder))
+                {
+                    return false;
+                }
+                if (type == typeof(PhotoAlbum))
+                {
+                    return false;
+                }
+                if (type == typeof(Season))
+                {
+                    return false;
+                }
+                if (type == typeof(MusicArtist))
+                {
+                    return false;
+                }
+            }
+            if (_config.Configuration.SkipDeserializationForPrograms)
+            {
+                if (type == typeof(LiveTvProgram))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        private BaseItem GetItem(IDataReader reader, InternalItemsQuery query)
+        {
             var typeString = reader.GetString(0);
 
             var type = _typeMapper.GetType(typeString);
@@ -1069,34 +1332,37 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
             BaseItem item = null;
 
-            using (var stream = reader.GetMemoryStream(1))
+            if (TypeRequiresDeserialization(type))
             {
-                try
-                {
-                    item = _jsonSerializer.DeserializeFromStream(stream, type) as BaseItem;
-                }
-                catch (SerializationException ex)
-                {
-                    Logger.ErrorException("Error deserializing item", ex);
-                }
-
-                if (item == null)
+                using (var stream = reader.GetMemoryStream(1, _memoryStreamProvider))
                 {
                     try
                     {
-                        item = Activator.CreateInstance(type) as BaseItem;
+                        item = _jsonSerializer.DeserializeFromStream(stream, type) as BaseItem;
                     }
-                    catch
+                    catch (SerializationException ex)
                     {
+                        Logger.ErrorException("Error deserializing item", ex);
                     }
                 }
+            }
 
-                if (item == null)
+            if (item == null)
+            {
+                try
+                {
+                    item = Activator.CreateInstance(type) as BaseItem;
+                }
+                catch
                 {
-                    return null;
                 }
             }
 
+            if (item == null)
+            {
+                return null;
+            }
+
             if (!reader.IsDBNull(2))
             {
                 var hasStartDate = item as IHasStartDate;
@@ -1170,194 +1436,275 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 }
             }
 
-            if (!reader.IsDBNull(15))
-            {
-                item.CommunityRating = reader.GetFloat(15);
-            }
+            var index = 15;
 
-            if (!reader.IsDBNull(16))
+            if (!reader.IsDBNull(index))
             {
-                item.CustomRating = reader.GetString(16);
+                item.CommunityRating = reader.GetFloat(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(17))
+            if (query.HasField(ItemFields.CustomRating))
             {
-                item.IndexNumber = reader.GetInt32(17);
+                if (!reader.IsDBNull(index))
+                {
+                    item.CustomRating = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(18))
+            if (!reader.IsDBNull(index))
             {
-                item.IsLocked = reader.GetBoolean(18);
+                item.IndexNumber = reader.GetInt32(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(19))
+            if (query.HasField(ItemFields.Settings))
             {
-                item.PreferredMetadataLanguage = reader.GetString(19);
-            }
+                if (!reader.IsDBNull(index))
+                {
+                    item.IsLocked = reader.GetBoolean(index);
+                }
+                index++;
 
-            if (!reader.IsDBNull(20))
-            {
-                item.PreferredMetadataCountryCode = reader.GetString(20);
+                if (!reader.IsDBNull(index))
+                {
+                    item.PreferredMetadataLanguage = reader.GetString(index);
+                }
+                index++;
+
+                if (!reader.IsDBNull(index))
+                {
+                    item.PreferredMetadataCountryCode = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(21))
+            if (!reader.IsDBNull(index))
             {
-                item.IsHD = reader.GetBoolean(21);
+                item.IsHD = reader.GetBoolean(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(22))
+            if (!reader.IsDBNull(index))
             {
-                item.ExternalEtag = reader.GetString(22);
+                item.ExternalEtag = reader.GetString(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(23))
+            if (!reader.IsDBNull(index))
             {
-                item.DateLastRefreshed = reader.GetDateTime(23).ToUniversalTime();
+                item.DateLastRefreshed = reader.GetDateTime(index).ToUniversalTime();
             }
+            index++;
 
-            if (!reader.IsDBNull(24))
+            if (!reader.IsDBNull(index))
             {
-                item.Name = reader.GetString(24);
+                item.Name = reader.GetString(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(25))
+            if (!reader.IsDBNull(index))
             {
-                item.Path = reader.GetString(25);
+                item.Path = reader.GetString(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(26))
+            if (!reader.IsDBNull(index))
             {
-                item.PremiereDate = reader.GetDateTime(26).ToUniversalTime();
+                item.PremiereDate = reader.GetDateTime(index).ToUniversalTime();
             }
+            index++;
 
-            if (!reader.IsDBNull(27))
+            if (query.HasField(ItemFields.Overview))
             {
-                item.Overview = reader.GetString(27);
+                if (!reader.IsDBNull(index))
+                {
+                    item.Overview = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(28))
+            if (!reader.IsDBNull(index))
             {
-                item.ParentIndexNumber = reader.GetInt32(28);
+                item.ParentIndexNumber = reader.GetInt32(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(29))
+            if (!reader.IsDBNull(index))
             {
-                item.ProductionYear = reader.GetInt32(29);
+                item.ProductionYear = reader.GetInt32(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(30))
+            if (!reader.IsDBNull(index))
             {
-                item.OfficialRating = reader.GetString(30);
+                item.OfficialRating = reader.GetString(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(31))
+            if (query.HasField(ItemFields.OfficialRatingDescription))
             {
-                item.OfficialRatingDescription = reader.GetString(31);
+                if (!reader.IsDBNull(index))
+                {
+                    item.OfficialRatingDescription = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(32))
+            if (query.HasField(ItemFields.HomePageUrl))
             {
-                item.HomePageUrl = reader.GetString(32);
+                if (!reader.IsDBNull(index))
+                {
+                    item.HomePageUrl = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(33))
+            if (query.HasField(ItemFields.DisplayMediaType))
             {
-                item.DisplayMediaType = reader.GetString(33);
+                if (!reader.IsDBNull(index))
+                {
+                    item.DisplayMediaType = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(34))
+            if (query.HasField(ItemFields.SortName))
             {
-                item.ForcedSortName = reader.GetString(34);
+                if (!reader.IsDBNull(index))
+                {
+                    item.ForcedSortName = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(35))
+            if (!reader.IsDBNull(index))
             {
-                item.RunTimeTicks = reader.GetInt64(35);
+                item.RunTimeTicks = reader.GetInt64(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(36))
+            if (query.HasField(ItemFields.VoteCount))
             {
-                item.VoteCount = reader.GetInt32(36);
+                if (!reader.IsDBNull(index))
+                {
+                    item.VoteCount = reader.GetInt32(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(37))
+            if (query.HasField(ItemFields.DateCreated))
             {
-                item.DateCreated = reader.GetDateTime(37).ToUniversalTime();
+                if (!reader.IsDBNull(index))
+                {
+                    item.DateCreated = reader.GetDateTime(index).ToUniversalTime();
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(38))
+            if (!reader.IsDBNull(index))
             {
-                item.DateModified = reader.GetDateTime(38).ToUniversalTime();
+                item.DateModified = reader.GetDateTime(index).ToUniversalTime();
             }
+            index++;
 
-            item.Id = reader.GetGuid(39);
+            item.Id = reader.GetGuid(index);
+            index++;
 
-            if (!reader.IsDBNull(40))
+            if (query.HasField(ItemFields.Genres))
             {
-                item.Genres = reader.GetString(40).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                if (!reader.IsDBNull(index))
+                {
+                    item.Genres = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(41))
+            if (!reader.IsDBNull(index))
             {
-                item.ParentId = reader.GetGuid(41);
+                item.ParentId = reader.GetGuid(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(42))
+            if (!reader.IsDBNull(index))
             {
-                item.Audio = (ProgramAudio)Enum.Parse(typeof(ProgramAudio), reader.GetString(42), true);
+                item.Audio = (ProgramAudio)Enum.Parse(typeof(ProgramAudio), reader.GetString(index), true);
             }
+            index++;
 
-            if (!reader.IsDBNull(43))
+            if (query.HasField(ItemFields.ServiceName))
             {
-                item.ServiceName = reader.GetString(43);
+                if (!reader.IsDBNull(index))
+                {
+                    item.ServiceName = reader.GetString(index);
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(44))
+            if (!reader.IsDBNull(index))
             {
-                item.IsInMixedFolder = reader.GetBoolean(44);
+                item.IsInMixedFolder = reader.GetBoolean(index);
             }
+            index++;
 
-            if (!reader.IsDBNull(45))
+            if (!reader.IsDBNull(index))
             {
-                item.DateLastSaved = reader.GetDateTime(45).ToUniversalTime();
+                item.DateLastSaved = reader.GetDateTime(index).ToUniversalTime();
             }
+            index++;
 
-            if (!reader.IsDBNull(46))
+            if (query.HasField(ItemFields.Settings))
             {
-                item.LockedFields = reader.GetString(46).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => (MetadataFields)Enum.Parse(typeof(MetadataFields), i, true)).ToList();
+                if (!reader.IsDBNull(index))
+                {
+                    item.LockedFields = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => (MetadataFields)Enum.Parse(typeof(MetadataFields), i, true)).ToList();
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(47))
+            if (query.HasField(ItemFields.Studios))
             {
-                item.Studios = reader.GetString(47).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                if (!reader.IsDBNull(index))
+                {
+                    item.Studios = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(48))
+            if (query.HasField(ItemFields.Tags))
             {
-                item.Tags = reader.GetString(48).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                if (!reader.IsDBNull(index))
+                {
+                    item.Tags = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                }
+                index++;
             }
 
-            if (!reader.IsDBNull(49))
+            if (!reader.IsDBNull(index))
             {
-                item.SourceType = (SourceType)Enum.Parse(typeof(SourceType), reader.GetString(49), true);
+                item.SourceType = (SourceType)Enum.Parse(typeof(SourceType), reader.GetString(index), true);
             }
+            index++;
 
             var trailer = item as Trailer;
             if (trailer != null)
             {
-                if (!reader.IsDBNull(50))
+                if (!reader.IsDBNull(index))
                 {
-                    trailer.TrailerTypes = reader.GetString(50).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => (TrailerType)Enum.Parse(typeof(TrailerType), i, true)).ToList();
+                    trailer.TrailerTypes = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => (TrailerType)Enum.Parse(typeof(TrailerType), i, true)).ToList();
                 }
             }
+            index++;
 
-            var index = 51;
-
-            if (!reader.IsDBNull(index))
+            if (query.HasField(ItemFields.OriginalTitle))
             {
-                item.OriginalTitle = reader.GetString(index);
+                if (!reader.IsDBNull(index))
+                {
+                    item.OriginalTitle = reader.GetString(index);
+                }
+                index++;
             }
-            index++;
 
             var video = item as Video;
             if (video != null)
@@ -1369,12 +1716,15 @@ namespace MediaBrowser.Server.Implementations.Persistence
             }
             index++;
 
-            var folder = item as Folder;
-            if (folder != null && !reader.IsDBNull(index))
+            if (query.HasField(ItemFields.DateLastMediaAdded))
             {
-                folder.DateLastMediaAdded = reader.GetDateTime(index).ToUniversalTime();
+                var folder = item as Folder;
+                if (folder != null && !reader.IsDBNull(index))
+                {
+                    folder.DateLastMediaAdded = reader.GetDateTime(index).ToUniversalTime();
+                }
+                index++;
             }
-            index++;
 
             if (!reader.IsDBNull(index))
             {
@@ -1388,11 +1738,14 @@ namespace MediaBrowser.Server.Implementations.Persistence
             }
             index++;
 
-            if (!reader.IsDBNull(index))
+            if (query.HasField(ItemFields.CriticRatingSummary))
             {
-                item.CriticRatingSummary = reader.GetString(index);
+                if (!reader.IsDBNull(index))
+                {
+                    item.CriticRatingSummary = reader.GetString(index);
+                }
+                index++;
             }
-            index++;
 
             if (!reader.IsDBNull(index))
             {
@@ -1459,6 +1812,99 @@ namespace MediaBrowser.Server.Implementations.Persistence
             }
             index++;
 
+            if (!reader.IsDBNull(index))
+            {
+                item.InheritedTags = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+            }
+            index++;
+
+            if (!reader.IsDBNull(index))
+            {
+                item.ExternalSeriesId = reader.GetString(index);
+            }
+            index++;
+
+            if (query.HasField(ItemFields.ShortOverview))
+            {
+                if (!reader.IsDBNull(index))
+                {
+                    item.ShortOverview = reader.GetString(index);
+                }
+                index++;
+            }
+
+            if (query.HasField(ItemFields.Taglines))
+            {
+                if (!reader.IsDBNull(index))
+                {
+                    item.Tagline = reader.GetString(index);
+                }
+                index++;
+            }
+
+            if (query.HasField(ItemFields.Keywords))
+            {
+                if (!reader.IsDBNull(index))
+                {
+                    item.Keywords = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                }
+                index++;
+            }
+
+            if (!reader.IsDBNull(index))
+            {
+                DeserializeProviderIds(reader.GetString(index), item);
+            }
+            index++;
+
+            if (query.DtoOptions.EnableImages)
+            {
+                if (!reader.IsDBNull(index))
+                {
+                    DeserializeImages(reader.GetString(index), item);
+                }
+                index++;
+            }
+
+            if (query.HasField(ItemFields.ProductionLocations))
+            {
+                if (!reader.IsDBNull(index))
+                {
+                    item.ProductionLocations = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).ToList();
+                }
+                index++;
+            }
+
+            if (!reader.IsDBNull(index))
+            {
+                item.ThemeSongIds = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => new Guid(i)).ToList();
+            }
+            index++;
+
+            if (!reader.IsDBNull(index))
+            {
+                item.ThemeVideoIds = reader.GetString(index).Split('|').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => new Guid(i)).ToList();
+            }
+            index++;
+
+            if (string.IsNullOrWhiteSpace(item.Tagline))
+            {
+                var movie = item as Movie;
+                if (movie != null && movie.Taglines.Count > 0)
+                {
+                    movie.Tagline = movie.Taglines[0];
+                }
+            }
+
+            if (type == typeof(Person) && item.ProductionLocations.Count == 0)
+            {
+                var person = (Person)item;
+                if (!string.IsNullOrWhiteSpace(person.PlaceOfBirth))
+                {
+                    item.ProductionLocations = new List<string> { person.PlaceOfBirth };
+                }
+            }
+
             return item;
         }
 
@@ -1723,28 +2169,28 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 return true;
             }
 
-            if (query.SortBy != null && query.SortBy.Length > 0)
+            var sortingFields = query.SortBy.ToList();
+            sortingFields.AddRange(query.OrderBy.Select(i => i.Item1));
+
+            if (sortingFields.Contains(ItemSortBy.IsFavoriteOrLiked, StringComparer.OrdinalIgnoreCase))
             {
-                if (query.SortBy.Contains(ItemSortBy.IsFavoriteOrLiked, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-                if (query.SortBy.Contains(ItemSortBy.IsPlayed, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-                if (query.SortBy.Contains(ItemSortBy.IsUnplayed, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-                if (query.SortBy.Contains(ItemSortBy.PlayCount, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-                if (query.SortBy.Contains(ItemSortBy.DatePlayed, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
+                return true;
+            }
+            if (sortingFields.Contains(ItemSortBy.IsPlayed, StringComparer.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+            if (sortingFields.Contains(ItemSortBy.IsUnplayed, StringComparer.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+            if (sortingFields.Contains(ItemSortBy.PlayCount, StringComparer.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+            if (sortingFields.Contains(ItemSortBy.DatePlayed, StringComparer.OrdinalIgnoreCase))
+            {
+                return true;
             }
 
             if (query.IsFavoriteOrLiked.HasValue)
@@ -1775,10 +2221,52 @@ namespace MediaBrowser.Server.Implementations.Persistence
             return false;
         }
 
+        private List<ItemFields> allFields = Enum.GetNames(typeof(ItemFields))
+                    .Select(i => (ItemFields)Enum.Parse(typeof(ItemFields), i, true))
+                    .ToList();
+
+        private IEnumerable<string> GetColumnNamesFromField(ItemFields field)
+        {
+            if (field == ItemFields.Settings)
+            {
+                return new[] { "IsLocked", "PreferredMetadataCountryCode", "PreferredMetadataLanguage", "LockedFields" };
+            }
+            if (field == ItemFields.ServiceName)
+            {
+                return new[] { "ExternalServiceId" };
+            }
+            if (field == ItemFields.SortName)
+            {
+                return new[] { "ForcedSortName" };
+            }
+            if (field == ItemFields.Taglines)
+            {
+                return new[] { "Tagline" };
+            }
+
+            return new[] { field.ToString() };
+        }
+
         private string[] GetFinalColumnsToSelect(InternalItemsQuery query, string[] startColumns, IDbCommand cmd)
         {
             var list = startColumns.ToList();
 
+            foreach (var field in allFields)
+            {
+                if (!query.HasField(field))
+                {
+                    foreach (var fieldToRemove in GetColumnNamesFromField(field).ToList())
+                    {
+                        list.Remove(fieldToRemove);
+                    }
+                }
+            }
+
+            if (!query.DtoOptions.EnableImages)
+            {
+                list.Remove("Images");
+            }
+
             if (EnableJoinUserData(query))
             {
                 list.Add("UserDataDb.UserData.UserId");
@@ -1933,7 +2421,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
                     while (reader.Read())
                     {
-                        var item = GetItem(reader);
+                        var item = GetItem(reader, query);
                         if (item != null)
                         {
                             list.Add(item);
@@ -2120,7 +2608,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
                     {
                         while (reader.Read())
                         {
-                            var item = GetItem(reader);
+                            var item = GetItem(reader, query);
                             if (item != null)
                             {
                                 list.Add(item);
@@ -2144,34 +2632,41 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
         private string GetOrderByText(InternalItemsQuery query)
         {
+            var orderBy = query.OrderBy.ToList();
+            var enableOrderInversion = true;
+
+            if (orderBy.Count == 0)
+            {
+                orderBy.AddRange(query.SortBy.Select(i => new Tuple<string, SortOrder>(i, query.SortOrder)));
+            }
+            else
+            {
+                enableOrderInversion = false;
+            }
+
             if (query.SimilarTo != null)
             {
-                if (query.SortBy == null || query.SortBy.Length == 0)
+                if (orderBy.Count == 0)
                 {
-                    if (query.User != null)
-                    {
-                        query.SortBy = new[] { "SimilarityScore", ItemSortBy.Random };
-                    }
-                    else
-                    {
-                        query.SortBy = new[] { "SimilarityScore", ItemSortBy.Random };
-                    }
+                    orderBy.Add(new Tuple<string, SortOrder>("SimilarityScore", SortOrder.Descending));
+                    orderBy.Add(new Tuple<string, SortOrder>(ItemSortBy.Random, SortOrder.Ascending));
                     query.SortOrder = SortOrder.Descending;
+                    enableOrderInversion = false;
                 }
             }
 
-            if (query.SortBy == null || query.SortBy.Length == 0)
+            query.OrderBy = orderBy;
+
+            if (orderBy.Count == 0)
             {
                 return string.Empty;
             }
 
-            var isAscending = query.SortOrder != SortOrder.Descending;
-
-            return " ORDER BY " + string.Join(",", query.SortBy.Select(i =>
+            return " ORDER BY " + string.Join(",", orderBy.Select(i =>
             {
-                var columnMap = MapOrderByField(i, query);
-                var columnAscending = isAscending;
-                if (columnMap.Item2)
+                var columnMap = MapOrderByField(i.Item1, query);
+                var columnAscending = i.Item2 == SortOrder.Ascending;
+                if (columnMap.Item2 && enableOrderInversion)
                 {
                     columnAscending = !columnAscending;
                 }
@@ -2523,38 +3018,116 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 whereClauses.Add("IsOffline=@IsOffline");
                 cmd.Parameters.Add(cmd, "@IsOffline", DbType.Boolean).Value = query.IsOffline;
             }
-            if (query.IsMovie.HasValue)
+
+            var exclusiveProgramAttribtues = !(query.IsMovie ?? true) ||
+                                             !(query.IsSports ?? true) ||
+                                             !(query.IsKids ?? true) ||
+                                             !(query.IsNews ?? true) ||
+                                             !(query.IsSeries ?? true);
+
+            if (exclusiveProgramAttribtues)
             {
-                var alternateTypes = new List<string>();
-                if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Movie).Name))
+                if (query.IsMovie.HasValue)
+                {
+                    var alternateTypes = new List<string>();
+                    if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Movie).Name))
+                    {
+                        alternateTypes.Add(typeof(Movie).FullName);
+                    }
+                    if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Trailer).Name))
+                    {
+                        alternateTypes.Add(typeof(Trailer).FullName);
+                    }
+
+                    if (alternateTypes.Count == 0)
+                    {
+                        whereClauses.Add("IsMovie=@IsMovie");
+                        cmd.Parameters.Add(cmd, "@IsMovie", DbType.Boolean).Value = query.IsMovie;
+                    }
+                    else
+                    {
+                        whereClauses.Add("(IsMovie is null OR IsMovie=@IsMovie)");
+                        cmd.Parameters.Add(cmd, "@IsMovie", DbType.Boolean).Value = query.IsMovie;
+                    }
+                }
+                if (query.IsSeries.HasValue)
                 {
-                    alternateTypes.Add(typeof(Movie).FullName);
+                    whereClauses.Add("IsSeries=@IsSeries");
+                    cmd.Parameters.Add(cmd, "@IsSeries", DbType.Boolean).Value = query.IsSeries;
                 }
-                if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Trailer).Name))
+                if (query.IsNews.HasValue)
                 {
-                    alternateTypes.Add(typeof(Trailer).FullName);
+                    whereClauses.Add("IsNews=@IsNews");
+                    cmd.Parameters.Add(cmd, "@IsNews", DbType.Boolean).Value = query.IsNews;
                 }
-
-                if (alternateTypes.Count == 0)
+                if (query.IsKids.HasValue)
                 {
-                    whereClauses.Add("IsMovie=@IsMovie");
+                    whereClauses.Add("IsKids=@IsKids");
+                    cmd.Parameters.Add(cmd, "@IsKids", DbType.Boolean).Value = query.IsKids;
                 }
-                else
+                if (query.IsSports.HasValue)
                 {
-                    whereClauses.Add("(IsMovie is null OR IsMovie=@IsMovie)");
+                    whereClauses.Add("IsSports=@IsSports");
+                    cmd.Parameters.Add(cmd, "@IsSports", DbType.Boolean).Value = query.IsSports;
                 }
-                cmd.Parameters.Add(cmd, "@IsMovie", DbType.Boolean).Value = query.IsMovie;
             }
-            if (query.IsKids.HasValue)
+            else
             {
-                whereClauses.Add("IsKids=@IsKids");
-                cmd.Parameters.Add(cmd, "@IsKids", DbType.Boolean).Value = query.IsKids;
+                var programAttribtues = new List<string>();
+                if (query.IsMovie ?? false)
+                {
+                    var alternateTypes = new List<string>();
+                    if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Movie).Name))
+                    {
+                        alternateTypes.Add(typeof(Movie).FullName);
+                    }
+                    if (query.IncludeItemTypes.Length == 0 || query.IncludeItemTypes.Contains(typeof(Trailer).Name))
+                    {
+                        alternateTypes.Add(typeof(Trailer).FullName);
+                    }
+
+                    if (alternateTypes.Count == 0)
+                    {
+                        programAttribtues.Add("IsMovie=@IsMovie");
+                    }
+                    else
+                    {
+                        programAttribtues.Add("(IsMovie is null OR IsMovie=@IsMovie)");
+                    }
+
+                    cmd.Parameters.Add(cmd, "@IsMovie", DbType.Boolean).Value = true;
+                }
+                if (query.IsSports ?? false)
+                {
+                    programAttribtues.Add("IsSports=@IsSports");
+                    cmd.Parameters.Add(cmd, "@IsSports", DbType.Boolean).Value = true;
+                }
+                if (query.IsNews ?? false)
+                {
+                    programAttribtues.Add("IsNews=@IsNews");
+                    cmd.Parameters.Add(cmd, "@IsNews", DbType.Boolean).Value = true;
+                }
+                if (query.IsSeries ?? false)
+                {
+                    programAttribtues.Add("IsSeries=@IsSeries");
+                    cmd.Parameters.Add(cmd, "@IsSeries", DbType.Boolean).Value = true;
+                }
+                if (query.IsKids ?? false)
+                {
+                    programAttribtues.Add("IsKids=@IsKids");
+                    cmd.Parameters.Add(cmd, "@IsKids", DbType.Boolean).Value = true;
+                }
+                if (programAttribtues.Count > 0)
+                {
+                    whereClauses.Add("(" + string.Join(" OR ", programAttribtues.ToArray()) + ")");
+                }
             }
-            if (query.IsSports.HasValue)
+
+            if (query.SimilarTo != null)
             {
-                whereClauses.Add("IsSports=@IsSports");
-                cmd.Parameters.Add(cmd, "@IsSports", DbType.Boolean).Value = query.IsSports;
+                whereClauses.Add("SimilarityScore > 0");
             }
+
             if (query.IsFolder.HasValue)
             {
                 whereClauses.Add("IsFolder=@IsFolder");
@@ -2626,6 +3199,18 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 cmd.Parameters.Add(cmd, "@MinIndexNumber", DbType.Int32).Value = query.MinIndexNumber.Value;
             }
 
+            if (query.MinDateCreated.HasValue)
+            {
+                whereClauses.Add("DateCreated>=@MinDateCreated");
+                cmd.Parameters.Add(cmd, "@MinDateCreated", DbType.DateTime).Value = query.MinDateCreated.Value;
+            }
+
+            if (query.MinDateLastSaved.HasValue)
+            {
+                whereClauses.Add("DateLastSaved>=@MinDateLastSaved");
+                cmd.Parameters.Add(cmd, "@MinDateLastSaved", DbType.DateTime).Value = query.MinDateLastSaved.Value;
+            }
+
             //if (query.MinPlayers.HasValue)
             //{
             //    whereClauses.Add("Players>=@MinPlayers");
@@ -2765,6 +3350,12 @@ namespace MediaBrowser.Server.Implementations.Persistence
                 cmd.Parameters.Add(cmd, "@MinSortName", DbType.String).Value = query.MinSortName;
             }
 
+            if (!string.IsNullOrWhiteSpace(query.ExternalSeriesId))
+            {
+                whereClauses.Add("ExternalSeriesId=@ExternalSeriesId");
+                cmd.Parameters.Add(cmd, "@ExternalSeriesId", DbType.String).Value = query.ExternalSeriesId;
+            }
+
             if (!string.IsNullOrWhiteSpace(query.Name))
             {
                 whereClauses.Add("CleanName=@Name");