GabrielTofvesson/MLAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimized byte serialization

Browse Source 
Fixed various bugs
Added option to pad misalignments for improved performance (at the possible cost of compression size if called in more than one non-consecutive instance)
This commit is contained in:
Gabriel Tofvesson 2018-04-13 16:28:30 +02:00
parent 902fb77ed4
commit 435db1f14b
2 changed files with 91 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MLAPI/NetworkingManagerComponents/Binary/BitReader.cs
View File

@ -35,6 +35,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary
bitCount += 8; bitCount += 8;
return result; return result;
} }
public void SkipPadded() => bitCount += (8 - (bitCount % 8)) % 8;
public ushort ReadUShort() => (ushort)ReadULong(); public ushort ReadUShort() => (ushort)ReadULong();
public uint ReadUInt() => (uint)ReadULong(); public uint ReadUInt() => (uint)ReadULong();
public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1); public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1);

179
MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs
View File

@ -1,6 +1,8 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Reflection; using System.Reflection;
using System.Runtime.InteropServices;
using System.Text; using System.Text;
using UnityEngine; using UnityEngine;
@ -52,7 +54,6 @@ namespace MLAPI.NetworkingManagerComponents.Binary
private List<object> collect = null; private List<object> collect = null;
private bool tempAlloc = false; private bool tempAlloc = false;
private int collectCount = 0;
/// <summary> /// <summary>
/// Allocates a new binary collector. /// Allocates a new binary collector.
@ -73,10 +74,11 @@ namespace MLAPI.NetworkingManagerComponents.Binary
private void Push<T>(T b) private void Push<T>(T b)
{ {
if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) if (b == null) collect.Add(b);
else if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object); collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object);
//else else
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
} }
@ -87,11 +89,12 @@ namespace MLAPI.NetworkingManagerComponents.Binary
public void WriteUShort(ushort s) => Push(s); public void WriteUShort(ushort s) => Push(s);
public void WriteUInt(uint i) => Push(i); public void WriteUInt(uint i) => Push(i);
public void WriteULong(ulong l) => Push(l); public void WriteULong(ulong l) => Push(l);
public void WriteSByte(sbyte b) => Push(b); public void WriteSByte(sbyte b) => Push(ZigZagEncode(b, 8));
public void WriteShort(short s) => Push(s); public void WriteShort(short s) => Push(ZigZagEncode(s, 8));
public void WriteInt(int i) => Push(i); public void WriteInt(int i) => Push(ZigZagEncode(i, 8));
public void WriteLong(long l) => Push(l); public void WriteLong(long l) => Push(ZigZagEncode(l, 8));
public void WriteString(string s) => Push(s); public void WriteString(string s) => Push(s);
public void WriteAlignBits() => Push<object>(null);
public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known); public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known);
public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known); public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known);
public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known); public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known);
@ -103,10 +106,12 @@ namespace MLAPI.NetworkingManagerComponents.Binary
public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known); public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known);
public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known); public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known);
private void PushArray<T>(T[] t, bool knownSize = false) public void PushArray<T>(T[] t, bool knownSize = false)
{ {
if (!knownSize) Push(t); if (!knownSize) Push((uint)t.Length);
else foreach (T t1 in t) Push(t1); bool signed = IsSigned(t.GetType().GetElementType());
int size = Marshal.SizeOf(t.GetType().GetElementType());
foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1);
} }
public long Finalize(ref byte[] buffer) public long Finalize(ref byte[] buffer)
@ -117,16 +122,22 @@ namespace MLAPI.NetworkingManagerComponents.Binary
return 0; return 0;
} }
long bitCount = 0; long bitCount = 0;
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
if(buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))) if (buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)))
{ {
Debug.LogWarning("MLAPI: The buffer size is not large enough"); Debug.LogWarning("MLAPI: The buffer size is not large enough");
return 0; return 0;
} }
long bitOffset = 0; long bitOffset = 0;
bool isAligned = true;
foreach (var item in collect) foreach (var item in collect)
Serialize(item, buffer, ref bitOffset); if (item == null)
{
bitOffset += (8 - (bitOffset % 8)) % 8;
isAligned = true;
}
else Serialize(item, buffer, ref bitOffset, ref isAligned);
return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1); return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1);
} }
@ -134,35 +145,36 @@ namespace MLAPI.NetworkingManagerComponents.Binary
public long GetFinalizeSize() public long GetFinalizeSize()
{ {
long bitCount = 0; long bitCount = 0;
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); for (int i = 0; i < collect.Count; ++i) bitCount += collect[i]==null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1));
} }
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset) private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset, ref bool isAligned)
{ {
Type type = t.GetType(); Type type = t.GetType();
bool size = false; bool size = false;
if (type.IsArray) if (type.IsArray)
{ {
var array = t as Array; var array = t as Array;
Serialize((uint)array.Length, writeTo, ref bitOffset); Serialize((uint)array.Length, writeTo, ref bitOffset, ref isAligned);
foreach (var element in array) foreach (var element in array)
Serialize(element, writeTo, ref bitOffset); Serialize(element, writeTo, ref bitOffset, ref isAligned);
} }
else if (IsSupportedType(type)) else if (IsSupportedType(type))
{ {
long offset = GetBitAllocation(type); long offset = t is bool ? 1 : BytesToRead(t) * 8;
if (type == typeof(bool)) if (type == typeof(bool))
{ {
WriteBit(writeTo, t as bool? ?? false, bitOffset); WriteBit(writeTo, t as bool? ?? false, bitOffset);
bitOffset += offset; bitOffset += offset;
isAligned = bitOffset % 8 == 0;
} }
else if (type == typeof(decimal)) else if (type == typeof(decimal))
{ {
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset); WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset, isAligned);
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32); WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32, isAligned);
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64); WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64, isAligned);
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96); WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96, isAligned);
bitOffset += offset; bitOffset += offset;
} }
else if ((size = type == typeof(float)) || type == typeof(double)) else if ((size = type == typeof(float)) || type == typeof(double))
@ -181,71 +193,75 @@ namespace MLAPI.NetworkingManagerComponents.Binary
// Since floating point flag bits are seemingly the highest bytes of the floating point values // Since floating point flag bits are seemingly the highest bytes of the floating point values
// and even very small values have them, we swap the endianness in the hopes of reducing the size // and even very small values have them, we swap the endianness in the hopes of reducing the size
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset); if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset); else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
} }
//bitOffset += offset;
} }
else else
{ {
bool signed = IsSigned(t.GetType()); //bool signed = IsSigned(t.GetType());
ulong value; ulong value;
if (signed) /*if (signed)
{ {
Type t1 = t.GetType(); Type t1 = t.GetType();
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1); if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1);
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2); else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2);
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4); else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4);
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8); else /*if (t1 == typeof(long)) value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
}
else*/
if (t is byte)
{
WriteByte(writeTo, t as byte? ?? 0, bitOffset, isAligned);
return;
} }
else if (t is byte) value = t as byte? ?? 0;
else if (t is ushort) value = t as ushort? ?? 0; else if (t is ushort) value = t as ushort? ?? 0;
else if (t is uint) value = t as uint? ?? 0; else if (t is uint) value = t as uint? ?? 0;
else /*if (t is ulong)*/ value = t as ulong? ?? 0; else /*if (t is ulong)*/ value = t as ulong? ?? 0;
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset); if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset, isAligned);
else if (value <= 2287) else if (value <= 2287)
{ {
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset); WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset, isAligned);
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8); WriteByte(writeTo, (value - 240) % 256, bitOffset + 8, isAligned);
} }
else if (value <= 67823) else if (value <= 67823)
{ {
WriteByte(writeTo, 249, bitOffset); WriteByte(writeTo, 249, bitOffset, isAligned);
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8); WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8, isAligned);
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16); WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16, isAligned);
} }
else else
{ {
WriteByte(writeTo, value & 255, bitOffset + 8); WriteByte(writeTo, value & 255, bitOffset + 8, isAligned);
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16); WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16, isAligned);
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24); WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24, isAligned);
if (value > 16777215) if (value > 16777215)
{ {
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32); WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32, isAligned);
if (value > 4294967295) if (value > 4294967295)
{ {
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40); WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40, isAligned);
if (value > 1099511627775) if (value > 1099511627775)
{ {
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48); WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48, isAligned);
if (value > 281474976710655) if (value > 281474976710655)
{ {
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56); WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56, isAligned);
if (value > 72057594037927935) if (value > 72057594037927935)
{ {
WriteByte(writeTo, 255, bitOffset); WriteByte(writeTo, 255, bitOffset, isAligned);
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64); WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64, isAligned);
} }
else WriteByte(writeTo, 254, bitOffset); else WriteByte(writeTo, 254, bitOffset, isAligned);
} }
else WriteByte(writeTo, 253, bitOffset); else WriteByte(writeTo, 253, bitOffset, isAligned);
} }
else WriteByte(writeTo, 252, bitOffset); else WriteByte(writeTo, 252, bitOffset, isAligned);
} }
else WriteByte(writeTo, 251, bitOffset); else WriteByte(writeTo, 251, bitOffset, isAligned);
} }
else WriteByte(writeTo, 250, bitOffset); else WriteByte(writeTo, 250, bitOffset, isAligned);
} }
bitOffset += BytesToRead(value) * 8; bitOffset += BytesToRead(value) * 8;
} }
@ -255,7 +271,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits)))); private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset)); private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null)); private static bool IsSigned(Type t) => t == typeof(sbyte) || t == typeof(short) || t == typeof(int) || t == typeof(long);
private static Type GetUnsignedType(Type t) => private static Type GetUnsignedType(Type t) =>
t == typeof(sbyte) ? typeof(byte) : t == typeof(sbyte) ? typeof(byte) :
@ -274,13 +290,16 @@ namespace MLAPI.NetworkingManagerComponents.Binary
{ {
Type elementType = type.GetElementType(); Type elementType = type.GetElementType();
count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements count += BytesToRead((t as Array).Length) * 8; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
foreach (var element in t as Array)
count += GetBitCount(element); if (elementType == typeof(bool)) count += (t as Array).Length;
else
foreach (var element in t as Array)
count += GetBitCount(element);
} }
else if (IsSupportedType(type)) else if (IsSupportedType(type))
{ {
long ba = GetBitAllocation(type); long ba = t is bool ? 1 : BytesToRead(t)*8;
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string); if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
else if (t is bool || t is decimal) count += ba; else if (t is bool || t is decimal) count += ba;
else count += BytesToRead(t) * 8; else count += BytesToRead(t) * 8;
@ -292,33 +311,32 @@ namespace MLAPI.NetworkingManagerComponents.Binary
private static void WriteBit(byte[] b, bool bit, long index) private static void WriteBit(byte[] b, bool bit, long index)
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0)); => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index); private static void WriteByte(byte[] b, ulong value, long index, bool isAligned) => WriteByte(b, (byte)value, index, isAligned);
private static void WriteByte(byte[] b, byte value, long index) private static void WriteByte(byte[] b, byte value, long index, bool isAligned)
{ {
int byteIndex = (int)(index / 8); if (isAligned) b[index / 8] = value;
int shift = (int)(index % 8); else
byte upper_mask = (byte)(0xFF << shift); {
byte lower_mask = (byte)~upper_mask; int byteIndex = (int)(index / 8);
int shift = (int)(index % 8);
byte upper_mask = (byte)(0xFF << shift);
b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift)); b[byteIndex] = (byte)((b[byteIndex] & (byte)~upper_mask) | (value << shift));
if(shift != 0 && byteIndex + 1 < b.Length)
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift))); b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
}
} }
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index) private static void WriteDynamic(byte[] b, int value, int byteCount, long index, bool isAligned)
{
for (int i = 0; i < bits; ++i)
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
}
private static void WriteDynamic(byte[] b, int value, int byteCount, long index)
{ {
for (int i = 0; i < byteCount; ++i) for (int i = 0; i < byteCount; ++i)
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i)); WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i), isAligned);
} }
private static int BytesToRead(object i) private static int BytesToRead(object i)
{ {
if (i is byte) return 1;
bool size; bool size;
ulong integer; ulong integer;
if (i is decimal) return BytesToRead((int)dec_flags.GetValue(i)) + BytesToRead((int)dec_lo.GetValue(i)) + BytesToRead((int)dec_mid.GetValue(i)) + BytesToRead((int)dec_hi.GetValue(i));
if ((size = i is float) || i is double) if ((size = i is float) || i is double)
{ {
int bytes = size ? 4 : 8; int bytes = size ? 4 : 8;
@ -337,7 +355,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary
else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)); else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0));
} }
} }
else integer = i as ulong? ?? 0; else integer = i as ulong? ?? i as uint? ?? i as ushort? ?? i as byte? ?? 0;
return return
integer <= 240 ? 1 : integer <= 240 ? 1 :
integer <= 2287 ? 2 : integer <= 2287 ? 2 :
@ -352,24 +370,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary
// Supported datatypes for serialization // Supported datatypes for serialization
private static bool IsSupportedType(Type t) => supportedTypes.Contains(t); private static bool IsSupportedType(Type t) => supportedTypes.Contains(t);
// Specifies how many bits will be written
private static long GetBitAllocation(Type t) =>
t == typeof(bool) ? 1 :
t == typeof(byte) ? 8 :
t == typeof(sbyte) ? 8 :
t == typeof(short) ? 16 :
t == typeof(char) ? 16 :
t == typeof(ushort) ? 16 :
t == typeof(int) ? 32 :
t == typeof(uint) ? 32 :
t == typeof(long) ? 64 :
t == typeof(ulong) ? 64 :
t == typeof(float) ? 32 :
t == typeof(double) ? 64 :
t == typeof(decimal) ? 128 :
0; // Unknown type
// Creates a weak reference to the allocated collector so that reuse may be possible // Creates a weak reference to the allocated collector so that reuse may be possible
public void Dispose() public void Dispose()
{ {