GabrielTofvesson/BankProject
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Fully implemented Binary Collector and Distributor

Browse Source 
- Added zigzag decoding
  - Added floating point value compression by reinterpreting and serializing floating point values as integer values
  - Moved serialization code to common code project
This commit is contained in:
Gabriel Tofvesson 2018-04-10 17:44:38 +02:00
parent 1da1b7878d
commit 308639da5f
9 changed files with 271 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
Client/BinaryDistributor.cs
View File

@ -1,68 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Client
{
public class BinaryDistributor
{
private static readonly byte[] holder = new byte[8];
private static readonly float[] holder_f = new float[1];
private static readonly double[] holder_d = new double[1];
private readonly byte[] readFrom;
private long bitCount = 0;
public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom;
public bool ReadBit()
{
bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0;
++bitCount;
return result;
}
public byte ReadByte()
{
int shift = (int)(bitCount % 8);
int index = (int)(bitCount / 8);
byte lower_mask = (byte)(0xFF << shift);
byte upper_mask = (byte)~lower_mask;
byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift)));
bitCount += 8;
return result;
}
public float ReadFloat() => ReadFloating<float>();
public double ReadDouble() => ReadFloating<double>();
public float[] ReadFloatArray() => ReadFloatingArray<float>();
public double[] ReadDoubleArray() => ReadFloatingArray<double>();
private T[] ReadFloatingArray<T>()
{
short size = (short)(ReadByte() | (ReadByte() << 8));
T[] result = new T[size];
for (short s = 0; s < size; ++s)
result[s] = ReadFloating<T>();
return result;
}
private T ReadFloating<T>()
{
int size = Marshal.SizeOf(typeof(T));
Array type_holder = size == 4 ? holder_f as Array: holder_d as Array;
T result;
lock(type_holder)
lock (holder)
{
for (int i = 0; i < size; ++i)
holder.SetValue(ReadByte(), i);
Buffer.BlockCopy(holder, 0, type_holder, 0, size);
result = (T) type_holder.GetValue(0);
}
return result;
}
}
}

2
Client/Client.csproj
View File

@ -43,8 +43,6 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BinaryCollector.cs" />
<Compile Include="BinaryDistributor.cs" />
<Compile Include="ConsoleForms\CancellationPipe.cs" />
<Compile Include="ConsoleForms\ConsoleController.cs" />
<Compile Include="ConsoleForms\Context.cs" />

12
Client/Program.cs
View File

@ -5,6 +5,7 @@ using Client;
using Client.ConsoleForms;
using Client.ConsoleForms.Parameters;
using Common;
using Tofvesson.Common;
namespace ConsoleForms
{
@ -19,19 +20,24 @@ namespace ConsoleForms
DebugStream = new TimeStampWriter(DebugStream, "HH:mm:ss.fff");
byte[] serialized;
using (BinaryCollector collector = new BinaryCollector(2))
using (BinaryCollector collector = new BinaryCollector(4))
{
collector.Push(true);
collector.Push(new double[] { 6.0, 5.0 });
collector.Push(new double[] { 6.0, 123 });
collector.Push(new float[] { 512, 1.2f, 1.337f});
collector.Push(5);
serialized = collector.ToArray();
}
BinaryDistributor bd = new BinaryDistributor(serialized);
bool bit = bd.ReadBit();
double[] result = bd.ReadDoubleArray();
float[] f = bd.ReadFloatArray();
int number = bd.ReadInt();
Padding p = new AbsolutePadding(2, 2, 1, 1);

3
Client/Resources/Layout/Session.xml
View File

@ -23,7 +23,8 @@
padding_left="2"
padding_right="2"
padding_top="1"
padding_bottom="1">
padding_bottom="1"
border="2">
<Views>
<ButtonView id="history">
<Text>@string/SE_hist</Text>

156
Client/BinaryCollector.cs → Common/BinaryCollector.cs
View File

@ -6,8 +6,9 @@ using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using Tofvesson.Crypto;
namespace Client
namespace Tofvesson.Common
{
public sealed class BinaryCollector : IDisposable
{
@ -17,6 +18,8 @@ namespace Client
private static readonly byte[] holder = new byte[8];
private static readonly float[] holder_f = new float[1];
private static readonly double[] holder_d = new double[1];
private static readonly ulong[] holder_u = new ulong[1];
private static readonly uint[] holder_i = new uint[1];
private static readonly List<Type> supportedTypes = new List<Type>()
{
typeof(bool),
@ -100,45 +103,124 @@ namespace Client
if (type.IsArray)
{
var array = t as Array;
Serialize((short)array.Length, writeTo, ref bitOffset);
Serialize((ushort)array.Length, writeTo, ref bitOffset);
foreach (var element in array)
Serialize(element, writeTo, ref bitOffset);
}
else if (IsSupportedType(type))
{
long offset = GetBitAllocation(type);
if (type == typeof(bool)) WriteBit(writeTo, t as bool? ?? false, bitOffset);
else if(type == typeof(decimal))
if (type == typeof(bool))
{
WriteBit(writeTo, t as bool? ?? false, bitOffset);
bitOffset += offset;
}
else if (type == typeof(decimal))
{
WriteDynamic(writeTo, dec_lo.GetValue(t), 4, bitOffset);
WriteDynamic(writeTo, dec_mid.GetValue(t), 4, bitOffset + 32);
WriteDynamic(writeTo, dec_hi.GetValue(t), 4, bitOffset + 64);
WriteDynamic(writeTo, dec_flags.GetValue(t), 4, bitOffset + 96);
bitOffset += offset;
}
else if((size = type == typeof(float)) || type == typeof(double))
else if ((size = type == typeof(float)) || type == typeof(double))
{
int bytes = size ? 4 : 8;
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
lock (type_holder)
lock (holder)
Array result_holder = size ? holder_i as Array : holder_u as Array;
lock (result_holder)
lock (type_holder)
{
// Clear artifacts
if (size) result_holder.SetValue(0U, 0);
else result_holder.SetValue(0UL, 0);
type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array
Buffer.BlockCopy(type_holder, 0, holder, 0, bytes); // Perform an internal copy to the byte-based holder
for (int i = 0; i < bytes; ++i)
WriteByte(writeTo, holder[i], bitOffset + (i * 8)); // Write the converted value to the output array
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
dynamic d = result_holder.GetValue(0);
// Since floating point flag bits are seemingly the highest bytes of the floating point values
// and even very small values have them, we swap the endianness in the hopes of reducing the size
Serialize(Support.SwapEndian(d), writeTo, ref bitOffset);
}
//bitOffset += offset;
}
else
{
dynamic value = t;
int type_size = Marshal.SizeOf(typeof(T));
for (int i = 0; i < type_size; ++i)
WriteByte(writeTo, (byte)(value >> (int)(i * 8)), bitOffset + (i * 8));
bool signed = IsSigned(t.GetType());
dynamic value;
if (signed)
{
Type t1 = t.GetType();
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t);
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t);
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t);
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t);
}
else value = t;
if (value <= 240) WriteByte(writeTo, value, bitOffset);
else if (value <= 2287)
{
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
}
else if (value <= 67823)
{
WriteByte(writeTo, 249, bitOffset);
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
}
else
{
WriteByte(writeTo, value & 255, bitOffset + 8);
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
if (value > 16777215)
{
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
if (value > 4294967295)
{
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
if (value > 1099511627775)
{
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
if (value > 281474976710655)
{
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
if (value > 72057594037927935)
{
WriteByte(writeTo, 255, bitOffset);
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
}
else WriteByte(writeTo, 254, bitOffset);
}
else WriteByte(writeTo, 253, bitOffset);
}
else WriteByte(writeTo, 252, bitOffset);
}
else WriteByte(writeTo, 251, bitOffset);
}
else WriteByte(writeTo, 250, bitOffset);
}
bitOffset += BytesToRead(value) * 8;
}
bitOffset += offset;
}
}
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
private static Type GetUnsignedType(Type t) =>
t == typeof(sbyte) ? typeof(byte) :
t == typeof(short) ? typeof(ushort) :
t == typeof(int) ? typeof(uint) :
t == typeof(long) ? typeof(ulong) :
null;
private static dynamic ZigZagEncode(dynamic d) => (((d >> (int)(Marshal.SizeOf(d) * 8 - 1))&1) | (d << 1));
private static long GetBitCount<T>(T t)
{
Type type = t.GetType();
@ -155,8 +237,8 @@ namespace Client
{
long ba = GetBitAllocation(type);
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
else if (t is bool || t is float || t is double || t is decimal) count += ba;
else count += BytesToRead(t, Marshal.SizeOf(t)) * 8;
else if (t is bool || t is decimal) count += ba;
else count += BytesToRead(t) * 8;
}
//else
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
@ -165,6 +247,7 @@ namespace Client
private static void WriteBit(byte[] b, bool bit, long index)
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
private static void WriteByte(byte[] b, dynamic value, long index) => WriteByte(b, (byte)value, index);
private static void WriteByte(byte[] b, byte value, long index)
{
int byteIndex = (int)(index / 8);
@ -176,18 +259,47 @@ namespace Client
if(shift != 0 && byteIndex + 1 < b.Length)
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
}
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
{
for (int i = 0; i < bits; ++i)
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
}
private static void WriteDynamic(byte[] b, dynamic value, int byteCount, long index)
{
for (int i = 0; i < byteCount; ++i)
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
}
private static int BytesToRead(dynamic integer, int maxBytes)
private static int BytesToRead(dynamic integer)
{
for (int i = 0; i < maxBytes; ++i)
if ((integer >> (8 * i)) & 0xFF <= 127)
return i + 1;
return maxBytes;
bool size;
if((size=integer is float) || integer is double)
{
int bytes = size ? 4 : 8;
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
Array result_holder = size ? holder_i as Array : holder_u as Array;
lock (result_holder)
lock (type_holder)
{
// Clear artifacts
if (size) result_holder.SetValue(0U, 0);
else result_holder.SetValue(0UL, 0);
type_holder.SetValue(integer, 0); // Insert the value to convert into the preallocated holder array
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
integer = Support.SwapEndian(integer = result_holder.GetValue(0));
}
}
return
integer <= 240 ? 1 :
integer <= 2287 ? 2 :
integer <= 67823 ? 3 :
integer <= 16777215 ? 4 :
integer <= 4294967295 ? 5 :
integer <= 1099511627775 ? 6 :
integer <= 281474976710655 ? 7 :
integer <= 72057594037927935 ? 8 :
9;
}
// Supported datatypes for serialization

108
Common/BinaryDistributor.cs Normal file
View File

@ -0,0 +1,108 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using Tofvesson.Crypto;
namespace Tofvesson.Common
{
public class BinaryDistributor
{
private static readonly byte[] holder = new byte[8];
private static readonly float[] holder_f = new float[1];
private static readonly double[] holder_d = new double[1];
private static readonly ulong[] holder_u = new ulong[1];
private static readonly uint[] holder_i = new uint[1];
private readonly byte[] readFrom;
private long bitCount = 0;
public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom;
public bool ReadBit()
{
bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0;
++bitCount;
return result;
}
public byte ReadByte()
{
int shift = (int)(bitCount % 8);
int index = (int)(bitCount / 8);
byte lower_mask = (byte)(0xFF << shift);
byte upper_mask = (byte)~lower_mask;
byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift)));
bitCount += 8;
return result;
}
public float ReadFloat() => ReadFloating<float>();
public double ReadDouble() => ReadFloating<double>();
public float[] ReadFloatArray() => ReadFloatingArray<float>();
public double[] ReadDoubleArray() => ReadFloatingArray<double>();
public ushort ReadUShort() => ReadUnsigned<ushort>();
public uint ReadUInt() => ReadUnsigned<uint>();
public ulong ReadULong() => ReadUnsigned<ulong>();
public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte());
public short ReadShort() => (short)ZigZagDecode(ReadUShort());
public int ReadInt() => (int)ZigZagDecode(ReadUInt());
public long ReadLong() => (long)ZigZagDecode(ReadULong());
private T ReadUnsigned<T>()
{
dynamic header = ReadByte();
if (header <= 240) return (T) header;
if (header <= 248) return (T) (240 + 256 * (header - 241) + ReadByte());
if (header == 249) return (T) (header = 2288 + 256 * ReadByte() + ReadByte());
dynamic res = ReadByte() | ((long)ReadByte() << 8) | ((long)ReadByte() << 16);
if(header > 250)
{
res |= (long) ReadByte() << 24;
if(header > 251)
{
res |= (long)ReadByte() << 32;
if(header > 252)
{
res |= (long)ReadByte() << 40;
if (header > 253)
{
res |= (long)ReadByte() << 48;
if (header > 254) res |= (long)ReadByte() << 56;
}
}
}
}
return (T) res;
}
private T[] ReadFloatingArray<T>()
{
ushort size = ReadUShort();
T[] result = new T[size];
for (short s = 0; s < size; ++s)
result[s] = ReadFloating<T>();
return result;
}
private T ReadFloating<T>()
{
int size = Marshal.SizeOf(typeof(T));
Array type_holder = size == 4 ? holder_f as Array : holder_d as Array;
Array result_holder = size == 4 ? holder_i as Array : holder_u as Array;
T result;
lock(result_holder)
lock (type_holder)
{
//for (int i = 0; i < size; ++i)
// holder.SetValue(ReadByte(), i);
if (size == 4) result_holder.SetValue(Support.SwapEndian(ReadUInt()), 0);
else result_holder.SetValue(Support.SwapEndian(ReadULong()), 0);
Buffer.BlockCopy(result_holder, 0, type_holder, 0, size);
result = (T)type_holder.GetValue(0);
}
return result;
}
private static dynamic ZigZagDecode(dynamic d) => (((d << (int)(Marshal.SizeOf(d) * 8 - 1)) & 1) | (d >> 1));
}
}

2
Common/Common.csproj
View File

@ -63,6 +63,8 @@
</ItemGroup>
<ItemGroup>
<Compile Include="AccountInfo.cs" />
<Compile Include="BinaryCollector.cs" />
<Compile Include="BinaryDistributor.cs" />
<Compile Include="Cryptography\AES.cs" />
<Compile Include="Cryptography\CBC.cs" />
<Compile Include="Collections.cs" />

9
Common/Support.cs
View File

@ -422,7 +422,14 @@ namespace Tofvesson.Crypto
// Swap endianness of a given integer
public static uint SwapEndian(uint value) => (uint)(((value >> 24) & (255 << 0)) | ((value >> 8) & (255 << 8)) | ((value << 8) & (255 << 16)) | ((value << 24) & (255 << 24)));
public static ulong SwapEndian(ulong value)
{
ulong res = 0;
for(int i = 0; i<8; ++i)
res = (res << 8) | ((value >> i * 8) & 0xFF);
return res;
}
public static ulong RightShift(this ulong value, int shift) => shift < 0 ? value << -shift : value >> shift;
public static string ToHexString(byte[] value)
{
StringBuilder builder = new StringBuilder();

8
Server/Program.cs
View File

@ -101,6 +101,14 @@ namespace Server
server.StopRunning();
}
private static void HandleInput()
{
while (true)
{
}
}
private static string[] ParseCommand(string cmd, out long id)
{
int idx = cmd.IndexOf(':'), idx1;