merge common implementations and server implementations

1 files changed, 640 insertions, 0 deletions

diff --git a/Emby.Server.Implementations/TextEncoding/UniversalDetector/Core/MBCSSM.cs b/Emby.Server.Implementations/TextEncoding/UniversalDetector/Core/MBCSSM.cs
new file mode 100644
index 000000000..7aa8581bc
--- /dev/null
+++ b/Emby.Server.Implementations/TextEncoding/UniversalDetector/Core/MBCSSM.cs
@@ -0,0 +1,640 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *          Shy Shalom <shooshX@gmail.com>
+ *          Rudi Pettazzi <rudi.pettazzi@gmail.com> (C# port)
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+namespace UniversalDetector.Core
+{
+    public class UTF8SMModel : SMModel
+    {
+        private readonly static int[] UTF8_cls = {
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 00 - 07
+            BitPackage.Pack4bits(1,1,1,1,1,1,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 10 - 17 
+            BitPackage.Pack4bits(1,1,1,0,1,1,1,1),  // 18 - 1f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 20 - 27 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 28 - 2f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 30 - 37 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 38 - 3f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 40 - 47 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 48 - 4f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 50 - 57 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 58 - 5f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 60 - 67 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 68 - 6f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 70 - 77 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 78 - 7f 
+            BitPackage.Pack4bits(2,2,2,2,3,3,3,3),  // 80 - 87 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 88 - 8f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 90 - 97 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 98 - 9f 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // a0 - a7 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // a8 - af 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // b0 - b7 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // b8 - bf 
+            BitPackage.Pack4bits(0,0,6,6,6,6,6,6),  // c0 - c7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // c8 - cf 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // d0 - d7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // d8 - df 
+            BitPackage.Pack4bits(7,8,8,8,8,8,8,8),  // e0 - e7 
+            BitPackage.Pack4bits(8,8,8,8,8,9,8,8),  // e8 - ef 
+            BitPackage.Pack4bits(10,11,11,11,11,11,11,11),  // f0 - f7 
+            BitPackage.Pack4bits(12,13,13,13,14,15,0,0)   // f8 - ff 
+        };
+
+        private readonly static int[] UTF8_st = {
+            BitPackage.Pack4bits(ERROR,START,ERROR,ERROR,ERROR,ERROR,   12,   10),//00-07 
+            BitPackage.Pack4bits(    9,   11,    8,    7,    6,    5,    4,    3),//08-0f 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//10-17 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f 
+            BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//20-27 
+            BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME,ITSME),//28-2f 
+            BitPackage.Pack4bits(ERROR,ERROR,    5,    5,    5,    5,ERROR,ERROR),//30-37 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//38-3f 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,    5,    5,    5,ERROR,ERROR),//40-47 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//48-4f 
+            BitPackage.Pack4bits(ERROR,ERROR,    7,    7,    7,    7,ERROR,ERROR),//50-57 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//58-5f 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,    7,    7,ERROR,ERROR),//60-67 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//68-6f 
+            BitPackage.Pack4bits(ERROR,ERROR,    9,    9,    9,    9,ERROR,ERROR),//70-77 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//78-7f 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,    9,ERROR,ERROR),//80-87 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//88-8f 
+            BitPackage.Pack4bits(ERROR,ERROR,   12,   12,   12,   12,ERROR,ERROR),//90-97 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//98-9f 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,   12,ERROR,ERROR),//a0-a7 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//a8-af 
+            BitPackage.Pack4bits(ERROR,ERROR,   12,   12,   12,ERROR,ERROR,ERROR),//b0-b7 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR),//b8-bf 
+            BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,ERROR,ERROR),//c0-c7 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ERROR) //c8-cf  
+        };
+
+        private readonly static int[] UTF8CharLenTable = 
+            {0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6 };
+        
+        public UTF8SMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UTF8_cls),
+                         16,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UTF8_st),
+              UTF8CharLenTable, "UTF-8")
+        {
+
+        }
+    }
+    
+    public class GB18030SMModel : SMModel
+    {
+        private readonly static int[] GB18030_cls = {
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 00 - 07 
+            BitPackage.Pack4bits(1,1,1,1,1,1,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 10 - 17 
+            BitPackage.Pack4bits(1,1,1,0,1,1,1,1),  // 18 - 1f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 20 - 27 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 28 - 2f 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // 30 - 37 
+            BitPackage.Pack4bits(3,3,1,1,1,1,1,1),  // 38 - 3f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 40 - 47 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 48 - 4f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 50 - 57 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 58 - 5f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 60 - 67 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 68 - 6f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 70 - 77 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,4),  // 78 - 7f 
+            BitPackage.Pack4bits(5,6,6,6,6,6,6,6),  // 80 - 87 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // 88 - 8f 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // 90 - 97 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // 98 - 9f 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // a0 - a7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // a8 - af 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // b0 - b7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // b8 - bf 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // c0 - c7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // c8 - cf 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // d0 - d7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // d8 - df 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // e0 - e7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // e8 - ef 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,6),  // f0 - f7 
+            BitPackage.Pack4bits(6,6,6,6,6,6,6,0)   // f8 - ff 
+        };
+
+        private readonly static int[] GB18030_st = {
+            BitPackage.Pack4bits(ERROR,START,START,START,START,START,    3,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START),//10-17 
+            BitPackage.Pack4bits(    4,ERROR,START,START,ERROR,ERROR,ERROR,ERROR),//18-1f 
+            BitPackage.Pack4bits(ERROR,ERROR,    5,ERROR,ERROR,ERROR,ITSME,ERROR),//20-27 
+            BitPackage.Pack4bits(ERROR,ERROR,START,START,START,START,START,START) //28-2f 
+        };
+
+        // To be accurate, the length of class 6 can be either 2 or 4. 
+        // But it is not necessary to discriminate between the two since 
+        // it is used for frequency analysis only, and we are validating 
+        // each code range there as well. So it is safe to set it to be 
+        // 2 here. 
+        private readonly static int[] GB18030CharLenTable = {0, 1, 1, 1, 1, 1, 2};
+        
+        public GB18030SMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, GB18030_cls),
+                         7,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, GB18030_st),
+              GB18030CharLenTable, "GB18030")
+        {
+
+        }
+    }
+    
+    public class BIG5SMModel : SMModel
+    {
+        private readonly static int[] BIG5_cls = {
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 00 - 07
+            BitPackage.Pack4bits(1,1,1,1,1,1,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 10 - 17 
+            BitPackage.Pack4bits(1,1,1,0,1,1,1,1),  // 18 - 1f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 20 - 27 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 28 - 2f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 30 - 37 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 38 - 3f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 40 - 47 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 48 - 4f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 50 - 57 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 58 - 5f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 60 - 67 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 68 - 6f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 70 - 77 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,1),  // 78 - 7f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 80 - 87 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 88 - 8f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 90 - 97 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 98 - 9f 
+            BitPackage.Pack4bits(4,3,3,3,3,3,3,3),  // a0 - a7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // a8 - af 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // b0 - b7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // b8 - bf 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // c0 - c7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // c8 - cf 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // d0 - d7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // d8 - df 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // e0 - e7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // e8 - ef 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // f0 - f7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,0)   // f8 - ff 
+        };
+
+        private readonly static int[] BIG5_st = {
+            BitPackage.Pack4bits(ERROR,START,START,    3,ERROR,ERROR,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ITSME,ITSME,ITSME,ITSME,ITSME,ERROR),//08-0f 
+            BitPackage.Pack4bits(ERROR,START,START,START,START,START,START,START) //10-17 
+        };
+
+        private readonly static int[] BIG5CharLenTable = {0, 1, 1, 2, 0};
+        
+        public BIG5SMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, BIG5_cls),
+                         5,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, BIG5_st),
+              BIG5CharLenTable, "Big5")
+        {
+
+        }
+    }
+    
+    public class EUCJPSMModel : SMModel
+    {
+        private readonly static int[] EUCJP_cls = {
+            //BitPacket.Pack4bits(5,4,4,4,4,4,4,4),  // 00 - 07 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 00 - 07 
+            BitPackage.Pack4bits(4,4,4,4,4,4,5,5),  // 08 - 0f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 10 - 17 
+            BitPackage.Pack4bits(4,4,4,5,4,4,4,4),  // 18 - 1f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 20 - 27 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 28 - 2f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 30 - 37 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 38 - 3f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 40 - 47 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 48 - 4f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 50 - 57 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 58 - 5f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 60 - 67 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 68 - 6f 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 70 - 77 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // 78 - 7f 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // 80 - 87 
+            BitPackage.Pack4bits(5,5,5,5,5,5,1,3),  // 88 - 8f 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // 90 - 97 
+            BitPackage.Pack4bits(5,5,5,5,5,5,5,5),  // 98 - 9f 
+            BitPackage.Pack4bits(5,2,2,2,2,2,2,2),  // a0 - a7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // a8 - af 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b0 - b7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b8 - bf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // c0 - c7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // c8 - cf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d0 - d7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d8 - df 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e0 - e7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e8 - ef 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // f0 - f7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,5)   // f8 - ff 
+        };
+
+        private readonly static int[] EUCJP_st = {
+            BitPackage.Pack4bits(    3,    4,    3,    5,START,ERROR,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,START,ERROR,START,ERROR,ERROR,ERROR),//10-17 
+            BitPackage.Pack4bits(ERROR,ERROR,START,ERROR,ERROR,ERROR,    3,ERROR),//18-1f 
+            BitPackage.Pack4bits(    3,ERROR,ERROR,ERROR,START,START,START,START) //20-27 
+        };
+
+        private readonly static int[] EUCJPCharLenTable = { 2, 2, 2, 3, 1, 0 };
+        
+        public EUCJPSMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCJP_cls),
+                         6,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCJP_st),
+              EUCJPCharLenTable, "EUC-JP")
+        {
+
+        }
+    }
+    
+    public class EUCKRSMModel : SMModel
+    {
+        private readonly static int[] EUCKR_cls = {
+            //BitPacket.Pack4bits(0,1,1,1,1,1,1,1),  // 00 - 07 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 00 - 07 
+            BitPackage.Pack4bits(1,1,1,1,1,1,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 10 - 17 
+            BitPackage.Pack4bits(1,1,1,0,1,1,1,1),  // 18 - 1f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 20 - 27 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 28 - 2f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 30 - 37 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 38 - 3f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 40 - 47 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 48 - 4f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 50 - 57 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 58 - 5f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 60 - 67 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 68 - 6f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 70 - 77 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 78 - 7f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 80 - 87 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 88 - 8f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 90 - 97 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 98 - 9f 
+            BitPackage.Pack4bits(0,2,2,2,2,2,2,2),  // a0 - a7 
+            BitPackage.Pack4bits(2,2,2,2,2,3,3,3),  // a8 - af 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b0 - b7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b8 - bf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // c0 - c7 
+            BitPackage.Pack4bits(2,3,2,2,2,2,2,2),  // c8 - cf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d0 - d7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d8 - df 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // e0 - e7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // e8 - ef 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // f0 - f7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,0)   // f8 - ff 
+        };
+
+        private readonly static int[] EUCKR_st = {
+            BitPackage.Pack4bits(ERROR,START,    3,ERROR,ERROR,ERROR,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ERROR,ERROR,START,START) //08-0f 
+        };
+
+        private readonly static int[] EUCKRCharLenTable = { 0, 1, 2, 0 };
+        
+        public EUCKRSMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCKR_cls),
+                         4,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCKR_st),
+              EUCKRCharLenTable, "EUC-KR")
+        {
+
+        }
+    }
+    
+    public class EUCTWSMModel : SMModel
+    {
+        private readonly static int[] EUCTW_cls = {
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 00 - 07 
+            BitPackage.Pack4bits(2,2,2,2,2,2,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 10 - 17 
+            BitPackage.Pack4bits(2,2,2,0,2,2,2,2),  // 18 - 1f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 20 - 27 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 28 - 2f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 30 - 37 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 38 - 3f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 40 - 47 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 48 - 4f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 50 - 57 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 58 - 5f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 60 - 67 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 68 - 6f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 70 - 77 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 78 - 7f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 80 - 87 
+            BitPackage.Pack4bits(0,0,0,0,0,0,6,0),  // 88 - 8f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 90 - 97 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 98 - 9f 
+            BitPackage.Pack4bits(0,3,4,4,4,4,4,4),  // a0 - a7 
+            BitPackage.Pack4bits(5,5,1,1,1,1,1,1),  // a8 - af 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // b0 - b7 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // b8 - bf 
+            BitPackage.Pack4bits(1,1,3,1,3,3,3,3),  // c0 - c7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // c8 - cf 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // d0 - d7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // d8 - df 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // e0 - e7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // e8 - ef 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // f0 - f7 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,0)   // f8 - ff 
+        };
+
+        private readonly static int[] EUCTW_st = {
+            BitPackage.Pack4bits(ERROR,ERROR,START,    3,    3,    3,    4,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ERROR,ERROR,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,ITSME,ITSME,ITSME,ERROR,START,ERROR),//10-17 
+            BitPackage.Pack4bits(START,START,START,ERROR,ERROR,ERROR,ERROR,ERROR),//18-1f 
+            BitPackage.Pack4bits(    5,ERROR,ERROR,ERROR,START,ERROR,START,START),//20-27 
+            BitPackage.Pack4bits(START,ERROR,START,START,START,START,START,START) //28-2f 
+        };
+
+        private readonly static int[] EUCTWCharLenTable = { 0, 0, 1, 2, 2, 2, 3 };
+        
+        public EUCTWSMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCTW_cls),
+                         7,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, EUCTW_st),
+              EUCTWCharLenTable, "EUC-TW")
+        {
+
+        }
+    }    
+    
+    public class SJISSMModel : SMModel
+    {
+        private readonly static int[] SJIS_cls = {
+            //BitPacket.Pack4bits(0,1,1,1,1,1,1,1),  // 00 - 07 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 00 - 07 
+            BitPackage.Pack4bits(1,1,1,1,1,1,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 10 - 17 
+            BitPackage.Pack4bits(1,1,1,0,1,1,1,1),  // 18 - 1f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 20 - 27 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 28 - 2f 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 30 - 37 
+            BitPackage.Pack4bits(1,1,1,1,1,1,1,1),  // 38 - 3f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 40 - 47 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 48 - 4f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 50 - 57 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 58 - 5f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 60 - 67 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 68 - 6f 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // 70 - 77 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,1),  // 78 - 7f 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // 80 - 87 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // 88 - 8f 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // 90 - 97 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // 98 - 9f 
+            //0xa0 is illegal in sjis encoding, but some pages does 
+            //contain such byte. We need to be more error forgiven.
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // a0 - a7     
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // a8 - af 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b0 - b7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // b8 - bf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // c0 - c7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // c8 - cf 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d0 - d7 
+            BitPackage.Pack4bits(2,2,2,2,2,2,2,2),  // d8 - df 
+            BitPackage.Pack4bits(3,3,3,3,3,3,3,3),  // e0 - e7 
+            BitPackage.Pack4bits(3,3,3,3,3,4,4,4),  // e8 - ef 
+            BitPackage.Pack4bits(4,4,4,4,4,4,4,4),  // f0 - f7 
+            BitPackage.Pack4bits(4,4,4,4,4,0,0,0)   // f8 - ff 
+        };
+
+        private readonly static int[] SJIS_st = {
+            BitPackage.Pack4bits(ERROR,START,START,    3,ERROR,ERROR,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,ERROR,ERROR,START,START,START,START) //10-17        
+        };
+
+        private readonly static int[] SJISCharLenTable = { 0, 1, 1, 2, 0, 0 };
+        
+        public SJISSMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, SJIS_cls),
+                         6,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, SJIS_st),
+              SJISCharLenTable, "Shift_JIS")
+        {
+
+        }
+    }
+    
+    public class UCS2BESMModel : SMModel
+    {
+        private readonly static int[] UCS2BE_cls = {
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 00 - 07 
+            BitPackage.Pack4bits(0,0,1,0,0,2,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 10 - 17 
+            BitPackage.Pack4bits(0,0,0,3,0,0,0,0),  // 18 - 1f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 20 - 27 
+            BitPackage.Pack4bits(0,3,3,3,3,3,0,0),  // 28 - 2f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 30 - 37 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 38 - 3f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 40 - 47 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 48 - 4f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 50 - 57 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 58 - 5f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 60 - 67 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 68 - 6f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 70 - 77 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 78 - 7f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 80 - 87 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 88 - 8f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 90 - 97 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 98 - 9f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // a0 - a7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // a8 - af 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // b0 - b7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // b8 - bf 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // c0 - c7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // c8 - cf 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // d0 - d7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // d8 - df 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e0 - e7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e8 - ef 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // f0 - f7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,4,5)   // f8 - ff 
+        };
+
+        private readonly static int[] UCS2BE_st = {
+            BitPackage.Pack4bits(    5,    7,    7,ERROR,    4,    3,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,    6,    6,    6,    6,ERROR,ERROR),//10-17 
+            BitPackage.Pack4bits(    6,    6,    6,    6,    6,ITSME,    6,    6),//18-1f 
+            BitPackage.Pack4bits(    6,    6,    6,    6,    5,    7,    7,ERROR),//20-27 
+            BitPackage.Pack4bits(    5,    8,    6,    6,ERROR,    6,    6,    6),//28-2f 
+            BitPackage.Pack4bits(    6,    6,    6,    6,ERROR,ERROR,START,START) //30-37 
+        };
+
+        private readonly static int[] UCS2BECharLenTable = { 2, 2, 2, 0, 2, 2 };
+        
+        public UCS2BESMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UCS2BE_cls),
+                         6,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UCS2BE_st),
+              UCS2BECharLenTable, "UTF-16BE")
+        {
+
+        }
+    }
+    
+    public class UCS2LESMModel : SMModel
+    {
+        private readonly static int[] UCS2LE_cls = {
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 00 - 07 
+            BitPackage.Pack4bits(0,0,1,0,0,2,0,0),  // 08 - 0f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 10 - 17 
+            BitPackage.Pack4bits(0,0,0,3,0,0,0,0),  // 18 - 1f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 20 - 27 
+            BitPackage.Pack4bits(0,3,3,3,3,3,0,0),  // 28 - 2f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 30 - 37 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 38 - 3f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 40 - 47 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 48 - 4f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 50 - 57 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 58 - 5f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 60 - 67 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 68 - 6f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 70 - 77 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 78 - 7f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 80 - 87 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 88 - 8f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 90 - 97 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // 98 - 9f 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // a0 - a7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // a8 - af 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // b0 - b7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // b8 - bf 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // c0 - c7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // c8 - cf 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // d0 - d7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // d8 - df 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e0 - e7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // e8 - ef 
+            BitPackage.Pack4bits(0,0,0,0,0,0,0,0),  // f0 - f7 
+            BitPackage.Pack4bits(0,0,0,0,0,0,4,5)   // f8 - ff 
+        };
+
+        private readonly static int[] UCS2LE_st = {
+            BitPackage.Pack4bits(    6,    6,    7,    6,    4,    3,ERROR,ERROR),//00-07 
+            BitPackage.Pack4bits(ERROR,ERROR,ERROR,ERROR,ITSME,ITSME,ITSME,ITSME),//08-0f 
+            BitPackage.Pack4bits(ITSME,ITSME,    5,    5,    5,ERROR,ITSME,ERROR),//10-17 
+            BitPackage.Pack4bits(    5,    5,    5,ERROR,    5,ERROR,    6,    6),//18-1f 
+            BitPackage.Pack4bits(    7,    6,    8,    8,    5,    5,    5,ERROR),//20-27 
+            BitPackage.Pack4bits(    5,    5,    5,ERROR,ERROR,ERROR,    5,    5),//28-2f 
+            BitPackage.Pack4bits(    5,    5,    5,ERROR,    5,ERROR,START,START) //30-37 
+        };
+
+        private readonly static int[] UCS2LECharLenTable = { 2, 2, 2, 2, 2, 2 };
+        
+        public UCS2LESMModel() : base(
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UCS2LE_cls),
+                         6,
+              new BitPackage(BitPackage.INDEX_SHIFT_4BITS, 
+                         BitPackage.SHIFT_MASK_4BITS, 
+                         BitPackage.BIT_SHIFT_4BITS,
+                         BitPackage.UNIT_MASK_4BITS, UCS2LE_st),
+              UCS2LECharLenTable, "UTF-16LE")
+        {
+
+        }
+    }
+    
+    
+}