Merge pull request #7604 from Jellifi007/fixes-diactritics

Co-authored-by: Cody Robibero <cody@robibe.ro>
(cherry picked from commit 8d1d9734381472b301deb0118bbb8da2a769a65e)
Signed-off-by: crobibero <cody@robibe.ro>

16 files changed, 85 insertions, 14 deletions

diff --git a/Emby.Dlna/Service/BaseControlHandler.cs b/Emby.Dlna/Service/BaseControlHandler.cs
index 7bec2eb72..9c423b395 100644
--- a/Emby.Dlna/Service/BaseControlHandler.cs
+++ b/Emby.Dlna/Service/BaseControlHandler.cs
@@ -6,8 +6,8 @@ using System.IO;
 using System.Text;
 using System.Threading.Tasks;
 using System.Xml;
-using Diacritics.Extensions;
 using Emby.Dlna.Didl;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Configuration;
 using Microsoft.Extensions.Logging;
 
diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs
index cdc14c260..6c243050d 100644
--- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs
+++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs
@@ -11,7 +11,6 @@ using System.Linq;
 using System.Text;
 using System.Text.Json;
 using System.Threading;
-using Diacritics.Extensions;
 using Emby.Server.Implementations.Playlists;
 using Jellyfin.Data.Enums;
 using Jellyfin.Extensions;
diff --git a/Emby.Server.Implementations/Library/SearchEngine.cs b/Emby.Server.Implementations/Library/SearchEngine.cs
index 96702d152..60778a443 100644
--- a/Emby.Server.Implementations/Library/SearchEngine.cs
+++ b/Emby.Server.Implementations/Library/SearchEngine.cs
@@ -5,9 +5,9 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using Diacritics.Extensions;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Dto;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Library;
diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs
index 1fa8e570d..2358fe623 100644
--- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs
+++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs
@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
 using BlurHashSharp.SkiaSharp;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 using MediaBrowser.Common.Configuration;
 using MediaBrowser.Common.Extensions;
 using MediaBrowser.Controller.Drawing;
diff --git a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs
index 0f2d7e62d..15a79fa1f 100644
--- a/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs
+++ b/MediaBrowser.Controller/Entities/Audio/MusicArtist.cs
@@ -8,9 +8,9 @@ using System.Linq;
 using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
-using Diacritics.Extensions;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.Entities;
 using Microsoft.Extensions.Logging;
diff --git a/MediaBrowser.Controller/Entities/Audio/MusicGenre.cs b/MediaBrowser.Controller/Entities/Audio/MusicGenre.cs
index 73a25232e..7448d02ea 100644
--- a/MediaBrowser.Controller/Entities/Audio/MusicGenre.cs
+++ b/MediaBrowser.Controller/Entities/Audio/MusicGenre.cs
@@ -5,8 +5,8 @@
 using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Diacritics.Extensions;
 using Jellyfin.Data.Enums;
+using Jellyfin.Extensions;
 using Microsoft.Extensions.Logging;
 
 namespace MediaBrowser.Controller.Entities.Audio
diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs
index 2bb966d2c..5cee6ce40 100644
--- a/MediaBrowser.Controller/Entities/BaseItem.cs
+++ b/MediaBrowser.Controller/Entities/BaseItem.cs
@@ -11,7 +11,6 @@ using System.Text;
 using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
-using Diacritics.Extensions;
 using Jellyfin.Data.Entities;
 using Jellyfin.Data.Enums;
 using Jellyfin.Extensions;
diff --git a/MediaBrowser.Controller/Entities/Genre.cs b/MediaBrowser.Controller/Entities/Genre.cs
index 4be673237..ddf62dd4c 100644
--- a/MediaBrowser.Controller/Entities/Genre.cs
+++ b/MediaBrowser.Controller/Entities/Genre.cs
@@ -5,8 +5,8 @@
 using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Diacritics.Extensions;
 using Jellyfin.Data.Enums;
+using Jellyfin.Extensions;
 using Microsoft.Extensions.Logging;
 
 namespace MediaBrowser.Controller.Entities
diff --git a/MediaBrowser.Controller/Entities/Person.cs b/MediaBrowser.Controller/Entities/Person.cs
index 045c1b89f..7f265084f 100644
--- a/MediaBrowser.Controller/Entities/Person.cs
+++ b/MediaBrowser.Controller/Entities/Person.cs
@@ -5,7 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Providers;
 using Microsoft.Extensions.Logging;
 
diff --git a/MediaBrowser.Controller/Entities/Studio.cs b/MediaBrowser.Controller/Entities/Studio.cs
index c8feb1c94..a3736a4bf 100644
--- a/MediaBrowser.Controller/Entities/Studio.cs
+++ b/MediaBrowser.Controller/Entities/Studio.cs
@@ -5,7 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 using Microsoft.Extensions.Logging;
 
 namespace MediaBrowser.Controller.Entities
diff --git a/MediaBrowser.Controller/Library/NameExtensions.cs b/MediaBrowser.Controller/Library/NameExtensions.cs
index d2ed3465a..9d78b8b6c 100644
--- a/MediaBrowser.Controller/Library/NameExtensions.cs
+++ b/MediaBrowser.Controller/Library/NameExtensions.cs
@@ -3,7 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 
 namespace MediaBrowser.Controller.Library
 {
diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj
index 3cab4366d..2368706fe 100644
--- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj
+++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj
@@ -18,7 +18,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Diacritics" Version="3.3.10" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs
index 0c52d2673..6d767914f 100644
--- a/MediaBrowser.Providers/Manager/MetadataService.cs
+++ b/MediaBrowser.Providers/Manager/MetadataService.cs
@@ -8,7 +8,7 @@ using System.Linq;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Entities;
 using MediaBrowser.Controller.Entities.Audio;
diff --git a/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzArtistProvider.cs b/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzArtistProvider.cs
index 1feb7f4ea..906a42f36 100644
--- a/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzArtistProvider.cs
+++ b/MediaBrowser.Providers/Plugins/MusicBrainz/MusicBrainzArtistProvider.cs
@@ -13,7 +13,7 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Xml;
-using Diacritics.Extensions;
+using Jellyfin.Extensions;
 using MediaBrowser.Controller.Entities.Audio;
 using MediaBrowser.Controller.Providers;
 using MediaBrowser.Model.Entities;
diff --git a/src/Jellyfin.Extensions/StringExtensions.cs b/src/Jellyfin.Extensions/StringExtensions.cs
index 3a7707253..dadc9f1d5 100644
--- a/src/Jellyfin.Extensions/StringExtensions.cs
+++ b/src/Jellyfin.Extensions/StringExtensions.cs
@@ -1,4 +1,8 @@
 using System;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text;
+using System.Text.RegularExpressions;
 
 namespace Jellyfin.Extensions
 {
@@ -7,6 +11,44 @@ namespace Jellyfin.Extensions
     /// </summary>
     public static class StringExtensions
     {
+        // Matches non-conforming unicode chars
+        // https://mnaoumov.wordpress.com/2014/06/14/stripping-invalid-characters-from-utf-16-strings/
+        private static readonly Regex _nonConformingUnicode = new Regex("([\ud800-\udbff](?![\udc00-\udfff]))|((?<![\ud800-\udbff])[\udc00-\udfff])|(\ufffd)");
+
+        /// <summary>
+        /// Removes the diacritics character from the strings.
+        /// </summary>
+        /// <param name="text">The string to act on.</param>
+        /// <returns>The string without diacritics character.</returns>
+        public static string RemoveDiacritics(this string text)
+        {
+            string withDiactritics = _nonConformingUnicode
+                .Replace(text, string.Empty)
+                .Normalize(NormalizationForm.FormD);
+
+            var withoutDiactritics = new StringBuilder();
+            foreach (char c in withDiactritics)
+            {
+                UnicodeCategory uc = CharUnicodeInfo.GetUnicodeCategory(c);
+                if (uc != UnicodeCategory.NonSpacingMark)
+                {
+                    withoutDiactritics.Append(c);
+                }
+            }
+
+            return withoutDiactritics.ToString().Normalize(NormalizationForm.FormC);
+        }
+
+        /// <summary>
+        /// Checks wether or not the specified string has diacritics in it.
+        /// </summary>
+        /// <param name="text">The string to check.</param>
+        /// <returns>True if the string has diacritics, false otherwise.</returns>
+        public static bool HasDiacritics(this string text)
+        {
+            return !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal);
+        }
+
         /// <summary>
         /// Counts the number of occurrences of [needle] in the string.
         /// </summary>
diff --git a/tests/Jellyfin.Extensions.Tests/StringExtensionsTests.cs b/tests/Jellyfin.Extensions.Tests/StringExtensionsTests.cs
index 7186cc023..903d88caa 100644
--- a/tests/Jellyfin.Extensions.Tests/StringExtensionsTests.cs
+++ b/tests/Jellyfin.Extensions.Tests/StringExtensionsTests.cs
@@ -6,6 +6,38 @@ namespace Jellyfin.Extensions.Tests
     public class StringExtensionsTests
     {
         [Theory]
+        [InlineData("", "")] // Identity edge-case (no diactritics)
+        [InlineData("Indiana Jones", "Indiana Jones")] // Identity (no diactritics)
+        [InlineData("a\ud800b", "ab")] // Invalid UTF-16 char stripping
+        [InlineData("Jön", "Jon")] // Issue #7484
+        [InlineData("Jönssonligan", "Jonssonligan")] // Issue #7484
+        [InlineData("Kieślowski", "Kieslowski")] // Issue #7450
+        [InlineData("Cidadão Kane", "Cidadao Kane")] // Issue #7560
+        [InlineData("운명처럼 널 사랑해", "운명처럼 널 사랑해")] // Issue #6393 (Korean language support)
+        [InlineData("애타는 로맨스", "애타는 로맨스")] // Issue #6393
+        public void RemoveDiacritics_ValidInput_Corrects(string input, string expectedResult)
+        {
+            string result = input.RemoveDiacritics();
+            Assert.Equal(expectedResult, result);
+        }
+
+        [Theory]
+        [InlineData("", false)] // Identity edge-case (no diactritics)
+        [InlineData("Indiana Jones", false)] // Identity (no diactritics)
+        [InlineData("a\ud800b", true)] // Invalid UTF-16 char stripping
+        [InlineData("Jön", true)] // Issue #7484
+        [InlineData("Jönssonligan", true)] // Issue #7484
+        [InlineData("Kieślowski", true)] // Issue #7450
+        [InlineData("Cidadão Kane", true)] // Issue #7560
+        [InlineData("운명처럼 널 사랑해", false)] // Issue #6393 (Korean language support)
+        [InlineData("애타는 로맨스", false)] // Issue #6393
+        public void HasDiacritics_ValidInput_Corrects(string input, bool expectedResult)
+        {
+            bool result = input.HasDiacritics();
+            Assert.Equal(expectedResult, result);
+        }
+
+        [Theory]
         [InlineData("", '_', 0)]
         [InlineData("___", '_', 3)]
         [InlineData("test\x00", '\x00', 1)]