Fully implemented Binary Collector and Distributor

- Added zigzag decoding
  - Added floating point value compression by reinterpreting and serializing floating point values as integer values
  - Moved serialization code to common code project

Client/BinaryDistributor.cs

@@ -1,68 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Client
-{
-    public class BinaryDistributor
-    {
-        private static readonly byte[] holder = new byte[8];
-        private static readonly float[] holder_f = new float[1];
-        private static readonly double[] holder_d = new double[1];
-
-        private readonly byte[] readFrom;
-        private long bitCount = 0;
-        public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom;
-
-        public bool ReadBit()
-        {
-            bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0;
-            ++bitCount;
-            return result;
-        }
-
-        public byte ReadByte()
-        {
-            int shift = (int)(bitCount % 8);
-            int index = (int)(bitCount / 8);
-            byte lower_mask = (byte)(0xFF << shift);
-            byte upper_mask = (byte)~lower_mask;
-            byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift)));
-            bitCount += 8;
-            return result;
-        }
-
-        public float ReadFloat() => ReadFloating<float>();
-        public double ReadDouble() => ReadFloating<double>();
-        public float[] ReadFloatArray() => ReadFloatingArray<float>();
-        public double[] ReadDoubleArray() => ReadFloatingArray<double>();
-
-        private T[] ReadFloatingArray<T>()
-        {
-            short size = (short)(ReadByte() | (ReadByte() << 8));
-            T[] result = new T[size];
-            for (short s = 0; s < size; ++s)
-                result[s] = ReadFloating<T>();
-            return result;
-        }
-
-        private T ReadFloating<T>()
-        {
-            int size = Marshal.SizeOf(typeof(T));
-            Array type_holder = size == 4 ? holder_f as Array: holder_d as Array;
-            T result;
-            lock(type_holder)
-                lock (holder)
-                {
-                    for (int i = 0; i < size; ++i)
-                        holder.SetValue(ReadByte(), i);
-                    Buffer.BlockCopy(holder, 0, type_holder, 0, size);
-                    result = (T) type_holder.GetValue(0);
-                }
-            return result;
-        }
-    }
-}

Client/Client.csproj

@@ -43,8 +43,6 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="BinaryCollector.cs" />
-    <Compile Include="BinaryDistributor.cs" />
     <Compile Include="ConsoleForms\CancellationPipe.cs" />
     <Compile Include="ConsoleForms\ConsoleController.cs" />
     <Compile Include="ConsoleForms\Context.cs" />

Client/Program.cs

@@ -5,6 +5,7 @@ using Client;
 using Client.ConsoleForms;
 using Client.ConsoleForms.Parameters;
 using Common;
+using Tofvesson.Common;
 
 namespace ConsoleForms
 {
@@ -19,19 +20,24 @@ namespace ConsoleForms
             DebugStream = new TimeStampWriter(DebugStream, "HH:mm:ss.fff");
 
 
-
+            
             byte[] serialized;
 
-            using (BinaryCollector collector = new BinaryCollector(2))
+            
+            using (BinaryCollector collector = new BinaryCollector(4))
             {
                 collector.Push(true);
-                collector.Push(new double[] { 6.0, 5.0 });
+                collector.Push(new double[] { 6.0, 123 });
+                collector.Push(new float[] { 512, 1.2f, 1.337f});
+                collector.Push(5);
                 serialized = collector.ToArray();
             }
 
             BinaryDistributor bd = new BinaryDistributor(serialized);
             bool bit = bd.ReadBit();
             double[] result = bd.ReadDoubleArray();
+            float[] f = bd.ReadFloatArray();
+            int number = bd.ReadInt();
 
             Padding p = new AbsolutePadding(2, 2, 1, 1);
 

Client/Resources/Layout/Session.xml

@@ -23,7 +23,8 @@
     padding_left="2"
     padding_right="2"
     padding_top="1"
-    padding_bottom="1">
+    padding_bottom="1"
+    border="2">
     <Views>
       <ButtonView id="history">
         <Text>@string/SE_hist</Text>

Common/BinaryCollector.cs

@@ -6,8 +6,9 @@ using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
+using Tofvesson.Crypto;
 
-namespace Client
+namespace Tofvesson.Common
 {
     public sealed class BinaryCollector : IDisposable
     {
@@ -17,6 +18,8 @@ namespace Client
         private static readonly byte[] holder = new byte[8];
         private static readonly float[] holder_f = new float[1];
         private static readonly double[] holder_d = new double[1];
+        private static readonly ulong[] holder_u = new ulong[1];
+        private static readonly uint[] holder_i = new uint[1];
         private static readonly List<Type> supportedTypes = new List<Type>()
         {
             typeof(bool),
@@ -100,45 +103,124 @@ namespace Client
             if (type.IsArray)
             {
                 var array = t as Array;
-                Serialize((short)array.Length, writeTo, ref bitOffset);
+                Serialize((ushort)array.Length, writeTo, ref bitOffset);
                 foreach (var element in array)
                     Serialize(element, writeTo, ref bitOffset);
             }
             else if (IsSupportedType(type))
             {
                 long offset = GetBitAllocation(type);
-                if (type == typeof(bool)) WriteBit(writeTo, t as bool? ?? false, bitOffset);
+                if (type == typeof(bool))
-                else if(type == typeof(decimal))
+                {
+                    WriteBit(writeTo, t as bool? ?? false, bitOffset);
+                    bitOffset += offset;
+                }
+                else if (type == typeof(decimal))
                 {
                     WriteDynamic(writeTo, dec_lo.GetValue(t), 4, bitOffset);
                     WriteDynamic(writeTo, dec_mid.GetValue(t), 4, bitOffset + 32);
                     WriteDynamic(writeTo, dec_hi.GetValue(t), 4, bitOffset + 64);
                     WriteDynamic(writeTo, dec_flags.GetValue(t), 4, bitOffset + 96);
+                    bitOffset += offset;
                 }
-                else if((size = type == typeof(float)) || type == typeof(double))
+                else if ((size = type == typeof(float)) || type == typeof(double))
                 {
                     int bytes = size ? 4 : 8;
                     Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
-                    lock (type_holder)
+                    Array result_holder = size ? holder_i as Array : holder_u as Array;
-                        lock (holder)
+                    lock (result_holder)
+                        lock (type_holder)
                         {
+                            // Clear artifacts
+                            if (size) result_holder.SetValue(0U, 0);
+                            else result_holder.SetValue(0UL, 0);
                             type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array
-                            Buffer.BlockCopy(type_holder, 0, holder, 0, bytes); // Perform an internal copy to the byte-based holder
+                            Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
-                            for (int i = 0; i < bytes; ++i)
+                            dynamic d = result_holder.GetValue(0);
-                                WriteByte(writeTo, holder[i], bitOffset + (i * 8)); // Write the converted value to the output array
+
+                            // Since floating point flag bits are seemingly the highest bytes of the floating point values
+                            // and even very small values have them, we swap the endianness in the hopes of reducing the size
+                            Serialize(Support.SwapEndian(d), writeTo, ref bitOffset);
                         }
+                    //bitOffset += offset;
                 }
                 else
                 {
-                    dynamic value = t;
+                    bool signed = IsSigned(t.GetType());
-                    int type_size = Marshal.SizeOf(typeof(T));
+                    dynamic value;
-                    for (int i = 0; i < type_size; ++i)
+                    if (signed)
-                        WriteByte(writeTo, (byte)(value >> (int)(i * 8)), bitOffset + (i * 8));
+                    {
+                        Type t1 = t.GetType();
+                        if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t);
+                        else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t);
+                        else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t);
+                        else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t);
+                    }
+                    else value = t;
+
+                    if (value <= 240) WriteByte(writeTo, value, bitOffset);
+                    else if (value <= 2287)
+                    {
+                        WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
+                        WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
+                    }
+                    else if (value <= 67823)
+                    {
+                        WriteByte(writeTo, 249, bitOffset);
+                        WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
+                        WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
+                    }
+                    else
+                    {
+                        WriteByte(writeTo, value & 255, bitOffset + 8);
+                        WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
+                        WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
+                        if (value > 16777215)
+                        {
+                            WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
+                            if (value > 4294967295)
+                            {
+                                WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
+                                if (value > 1099511627775)
+                                {
+                                    WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
+                                    if (value > 281474976710655)
+                                    {
+                                        WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
+                                        if (value > 72057594037927935)
+                                        {
+                                            WriteByte(writeTo, 255, bitOffset);
+                                            WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
+                                        }
+                                        else WriteByte(writeTo, 254, bitOffset);
+                                    }
+                                    else WriteByte(writeTo, 253, bitOffset);
+                                }
+                                else WriteByte(writeTo, 252, bitOffset);
+                            }
+                            else WriteByte(writeTo, 251, bitOffset);
+                        }
+                        else WriteByte(writeTo, 250, bitOffset);
+                    }
+                    bitOffset += BytesToRead(value) * 8;
                 }
-                bitOffset += offset;
             }
         }
 
+        private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
+        private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
+
+        private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
+
+        private static Type GetUnsignedType(Type t) =>
+            t == typeof(sbyte) ? typeof(byte) :
+            t == typeof(short) ? typeof(ushort) :
+            t == typeof(int) ? typeof(uint) :
+            t == typeof(long) ? typeof(ulong) :
+            null;
+
+        private static dynamic ZigZagEncode(dynamic d) => (((d >> (int)(Marshal.SizeOf(d) * 8 - 1))&1) | (d << 1));
+
         private static long GetBitCount<T>(T t)
         {
             Type type = t.GetType();
@@ -155,8 +237,8 @@ namespace Client
             {
                 long ba = GetBitAllocation(type);
                 if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
-                else if (t is bool || t is float || t is double || t is decimal) count += ba;
+                else if (t is bool || t is decimal) count += ba;
-                else count += BytesToRead(t, Marshal.SizeOf(t)) * 8;
+                else count += BytesToRead(t) * 8;
             }
             //else
             //    Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
@@ -165,6 +247,7 @@ namespace Client
 
         private static void WriteBit(byte[] b, bool bit, long index)
             => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
+        private static void WriteByte(byte[] b, dynamic value, long index) => WriteByte(b, (byte)value, index);
         private static void WriteByte(byte[] b, byte value, long index)
         {
             int byteIndex = (int)(index / 8);
@@ -176,18 +259,47 @@ namespace Client
             if(shift != 0 && byteIndex + 1 < b.Length)
                 b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
         }
+        private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
+        {
+            for (int i = 0; i < bits; ++i)
+                WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
+        }
         private static void WriteDynamic(byte[] b, dynamic value, int byteCount, long index)
         {
             for (int i = 0; i < byteCount; ++i)
                 WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
         }
 
-        private static int BytesToRead(dynamic integer, int maxBytes)
+        private static int BytesToRead(dynamic integer)
         {
-            for (int i = 0; i < maxBytes; ++i)
+            bool size;
-                if ((integer >> (8 * i)) & 0xFF <= 127)
+            if((size=integer is float) || integer is double)
-                    return i + 1;
+            {
-            return maxBytes;
+                int bytes = size ? 4 : 8;
+                Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
+                Array result_holder = size ? holder_i as Array : holder_u as Array;
+                lock (result_holder)
+                    lock (type_holder)
+                    {
+                        // Clear artifacts
+                        if (size) result_holder.SetValue(0U, 0);
+                        else result_holder.SetValue(0UL, 0);
+
+                        type_holder.SetValue(integer, 0); // Insert the value to convert into the preallocated holder array
+                        Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
+                        integer = Support.SwapEndian(integer = result_holder.GetValue(0));
+                    }
+            }
+            return
+                integer <= 240 ? 1 :
+                integer <= 2287 ? 2 :
+                integer <= 67823 ? 3 :
+                integer <= 16777215 ? 4 :
+                integer <= 4294967295 ? 5 :
+                integer <= 1099511627775 ? 6 :
+                integer <= 281474976710655 ? 7 :
+                integer <= 72057594037927935 ? 8 :
+                9;
         }
 
         // Supported datatypes for serialization

Common/BinaryDistributor.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+using Tofvesson.Crypto;
+
+namespace Tofvesson.Common
+{
+    public class BinaryDistributor
+    {
+        private static readonly byte[] holder = new byte[8];
+        private static readonly float[] holder_f = new float[1];
+        private static readonly double[] holder_d = new double[1];
+        private static readonly ulong[] holder_u = new ulong[1];
+        private static readonly uint[] holder_i = new uint[1];
+
+        private readonly byte[] readFrom;
+        private long bitCount = 0;
+        public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom;
+
+        public bool ReadBit()
+        {
+            bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0;
+            ++bitCount;
+            return result;
+        }
+
+        public byte ReadByte()
+        {
+            int shift = (int)(bitCount % 8);
+            int index = (int)(bitCount / 8);
+            byte lower_mask = (byte)(0xFF << shift);
+            byte upper_mask = (byte)~lower_mask;
+            byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift)));
+            bitCount += 8;
+            return result;
+        }
+
+        public float ReadFloat() => ReadFloating<float>();
+        public double ReadDouble() => ReadFloating<double>();
+        public float[] ReadFloatArray() => ReadFloatingArray<float>();
+        public double[] ReadDoubleArray() => ReadFloatingArray<double>();
+        public ushort ReadUShort() => ReadUnsigned<ushort>();
+        public uint ReadUInt() => ReadUnsigned<uint>();
+        public ulong ReadULong() => ReadUnsigned<ulong>();
+        public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte());
+        public short ReadShort() => (short)ZigZagDecode(ReadUShort());
+        public int ReadInt() => (int)ZigZagDecode(ReadUInt());
+        public long ReadLong() => (long)ZigZagDecode(ReadULong());
+
+        private T ReadUnsigned<T>()
+        {
+            dynamic header = ReadByte();
+            if (header <= 240) return (T) header;
+            if (header <= 248) return (T) (240 + 256 * (header - 241) + ReadByte());
+            if (header == 249) return (T) (header = 2288 + 256 * ReadByte() + ReadByte());
+            dynamic res = ReadByte() | ((long)ReadByte() << 8) | ((long)ReadByte() << 16);
+            if(header > 250)
+            {
+                res |= (long) ReadByte() << 24;
+                if(header > 251)
+                {
+                    res |= (long)ReadByte() << 32;
+                    if(header > 252)
+                    {
+                        res |= (long)ReadByte() << 40;
+                        if (header > 253)
+                        {
+                            res |= (long)ReadByte() << 48;
+                            if (header > 254) res |= (long)ReadByte() << 56;
+                        }
+                    }
+                }
+            }
+            return (T) res;
+        }
+        private T[] ReadFloatingArray<T>()
+        {
+            ushort size = ReadUShort();
+            T[] result = new T[size];
+            for (short s = 0; s < size; ++s)
+                result[s] = ReadFloating<T>();
+            return result;
+        }
+
+        private T ReadFloating<T>()
+        {
+            int size = Marshal.SizeOf(typeof(T));
+            Array type_holder = size == 4 ? holder_f as Array : holder_d as Array;
+            Array result_holder = size == 4 ? holder_i as Array : holder_u as Array;
+            T result;
+            lock(result_holder)
+                lock (type_holder)
+                {
+                    //for (int i = 0; i < size; ++i)
+                    //    holder.SetValue(ReadByte(), i);
+                    if (size == 4) result_holder.SetValue(Support.SwapEndian(ReadUInt()), 0);
+                    else result_holder.SetValue(Support.SwapEndian(ReadULong()), 0);
+                    Buffer.BlockCopy(result_holder, 0, type_holder, 0, size);
+                    result = (T)type_holder.GetValue(0);
+                }
+            return result;
+        }
+        private static dynamic ZigZagDecode(dynamic d) => (((d << (int)(Marshal.SizeOf(d) * 8 - 1)) & 1) | (d >> 1));
+    }
+}

Common/Common.csproj

@@ -63,6 +63,8 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AccountInfo.cs" />
+    <Compile Include="BinaryCollector.cs" />
+    <Compile Include="BinaryDistributor.cs" />
     <Compile Include="Cryptography\AES.cs" />
     <Compile Include="Cryptography\CBC.cs" />
     <Compile Include="Collections.cs" />

Common/Support.cs

@@ -422,7 +422,14 @@ namespace Tofvesson.Crypto
         
         // Swap endianness of a given integer
         public static uint SwapEndian(uint value) => (uint)(((value >> 24) & (255 << 0)) | ((value >> 8) & (255 << 8)) | ((value << 8) & (255 << 16)) | ((value << 24) & (255 << 24)));
-
+        public static ulong SwapEndian(ulong value)
+        {
+            ulong res = 0;
+            for(int i = 0; i<8; ++i)
+                res = (res << 8) | ((value >> i * 8) & 0xFF);
+            return res;
+        }
+        public static ulong RightShift(this ulong value, int shift) => shift < 0 ? value << -shift : value >> shift;
         public static string ToHexString(byte[] value)
         {
             StringBuilder builder = new StringBuilder();

Server/Program.cs

@@ -101,6 +101,14 @@ namespace Server
             server.StopRunning();
         }
 
+        private static void HandleInput()
+        {
+            while (true)
+            {
+
+            }
+        }
+
         private static string[] ParseCommand(string cmd, out long id)
         {
             int idx = cmd.IndexOf(':'), idx1;