From baed45bad52a2e0f6ad24e30de04116e3906ba04 Mon Sep 17 00:00:00 2001 From: GabrielTofvesson Date: Fri, 6 Apr 2018 22:15:14 +0200 Subject: [PATCH 01/10] Added files? --- MLAPI/MLAPI.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MLAPI/MLAPI.csproj b/MLAPI/MLAPI.csproj index f0f9d50..5eb61bc 100644 --- a/MLAPI/MLAPI.csproj +++ b/MLAPI/MLAPI.csproj @@ -64,6 +64,8 @@ + + From 13470efd55b5573fa26b2b82415f3de21018ae8c Mon Sep 17 00:00:00 2001 From: GabrielTofvesson Date: Tue, 10 Apr 2018 21:05:13 +0200 Subject: [PATCH 02/10] Added new serialization/deserialization system - Code might look a bit janky because it was adapted from a version that used dynamic types --- MLAPI/MLAPI.csproj | 3 + .../Binary/BinaryCollector.cs | 334 ++++++++++++++++++ .../Binary/BinaryDistributor.cs | 106 ++++++ .../Binary/BinaryHelpers.cs | 22 ++ 4 files changed, 465 insertions(+) create mode 100644 MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs create mode 100644 MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs create mode 100644 MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs diff --git a/MLAPI/MLAPI.csproj b/MLAPI/MLAPI.csproj index f0b660f..9c7949f 100644 --- a/MLAPI/MLAPI.csproj +++ b/MLAPI/MLAPI.csproj @@ -78,6 +78,9 @@ + + + diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs new file mode 100644 index 0000000..80cb2f6 --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs @@ -0,0 +1,334 @@ +using MLAPI.NetworkingManagerComponents.Binary; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Tofvesson.Common +{ + public sealed class BinaryCollector : IDisposable + { + // Collects reusable + private static readonly List expired = new List(); + + private static readonly float[] holder_f = new float[1]; + private static readonly double[] holder_d = new double[1]; + private static readonly ulong[] holder_u = new ulong[1]; + private static readonly uint[] holder_i = new uint[1]; + private static readonly List supportedTypes = new List() + { + typeof(bool), + typeof(byte), + typeof(sbyte), + typeof(char), + typeof(short), + typeof(ushort), + typeof(int), + typeof(uint), + typeof(long), + typeof(ulong), + typeof(float), + typeof(double), + typeof(decimal) + }; + + private static readonly FieldInfo + dec_lo, + dec_mid, + dec_hi, + dec_flags; + + static BinaryCollector() + { + dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic); + dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic); + dec_hi = typeof(decimal).GetField("hi", BindingFlags.NonPublic); + dec_flags = typeof(decimal).GetField("flags", BindingFlags.NonPublic); + } + + private object[] collect; + private readonly int bufferSize; + private int collectCount = 0; + + /// + /// Allocates a new binary collector. + /// + public BinaryCollector(int bufferSize) + { + this.bufferSize = bufferSize; + for (int i = expired.Count - 1; i >= 0; --i) + if (expired[i].IsAlive) + { + collect = (object[])expired[i].Target; + if (collect.Length >= bufferSize) + { + expired.RemoveAt(i); // This entry he been un-expired for now + break; + } + } + else expired.RemoveAt(i); // Entry has been collected by GC + if (collect == null || collect.Length < bufferSize) + collect = new object[bufferSize]; + } + + public void Push(T b) + { + if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) + collect[collectCount++] = b is string ? Encoding.UTF8.GetBytes(b as string) : b as object; + //else + // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); + } + + public byte[] ToArray() + { + long bitCount = 0; + for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); + + byte[] alloc = new byte[(bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)]; + long bitOffset = 0; + foreach (var item in collect) + Serialize(item, alloc, ref bitOffset); + + return alloc; + } + + private static void Serialize(T t, byte[] writeTo, ref long bitOffset) + { + Type type = t.GetType(); + bool size = false; + if (type.IsArray) + { + var array = t as Array; + Serialize((ushort)array.Length, writeTo, ref bitOffset); + foreach (var element in array) + Serialize(element, writeTo, ref bitOffset); + } + else if (IsSupportedType(type)) + { + long offset = GetBitAllocation(type); + if (type == typeof(bool)) + { + WriteBit(writeTo, t as bool? ?? false, bitOffset); + bitOffset += offset; + } + else if (type == typeof(decimal)) + { + WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset); + WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32); + WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64); + WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96); + bitOffset += offset; + } + else if ((size = type == typeof(float)) || type == typeof(double)) + { + int bytes = size ? 4 : 8; + Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array + Array result_holder = size ? holder_i as Array : holder_u as Array; + lock (result_holder) + lock (type_holder) + { + // Clear artifacts + if (size) result_holder.SetValue(0U, 0); + else result_holder.SetValue(0UL, 0); + type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array + Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder + dynamic d = result_holder.GetValue(0); + + // Since floating point flag bits are seemingly the highest bytes of the floating point values + // and even very small values have them, we swap the endianness in the hopes of reducing the size + Serialize(BinaryHelpers.SwapEndian(d), writeTo, ref bitOffset); + } + //bitOffset += offset; + } + else + { + bool signed = IsSigned(t.GetType()); + dynamic value; + if (signed) + { + Type t1 = t.GetType(); + if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1); + else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2); + else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4); + else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8); + } + else value = t; + + if (value <= 240) WriteByte(writeTo, value, bitOffset); + else if (value <= 2287) + { + WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset); + WriteByte(writeTo, (value - 240) % 256, bitOffset + 8); + } + else if (value <= 67823) + { + WriteByte(writeTo, 249, bitOffset); + WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8); + WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16); + } + else + { + WriteByte(writeTo, value & 255, bitOffset + 8); + WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16); + WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24); + if (value > 16777215) + { + WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32); + if (value > 4294967295) + { + WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40); + if (value > 1099511627775) + { + WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48); + if (value > 281474976710655) + { + WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56); + if (value > 72057594037927935) + { + WriteByte(writeTo, 255, bitOffset); + WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64); + } + else WriteByte(writeTo, 254, bitOffset); + } + else WriteByte(writeTo, 253, bitOffset); + } + else WriteByte(writeTo, 252, bitOffset); + } + else WriteByte(writeTo, 251, bitOffset); + } + else WriteByte(writeTo, 250, bitOffset); + } + bitOffset += BytesToRead(value) * 8; + } + } + } + + private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits)))); + private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset)); + + private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null)); + + private static Type GetUnsignedType(Type t) => + t == typeof(sbyte) ? typeof(byte) : + t == typeof(short) ? typeof(ushort) : + t == typeof(int) ? typeof(uint) : + t == typeof(long) ? typeof(ulong) : + null; + + private static ulong ZigZagEncode(long d, int bytes) => (ulong)(((d >> (bytes * 8 - 1))&1) | (d << 1)); + + private static long GetBitCount(T t) + { + Type type = t.GetType(); + long count = 0; + if (type.IsArray) + { + Type elementType = type.GetElementType(); + + count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements + foreach (var element in t as Array) + count += GetBitCount(element); + } + else if (IsSupportedType(type)) + { + long ba = GetBitAllocation(type); + if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string); + else if (t is bool || t is decimal) count += ba; + else count += BytesToRead(t) * 8; + } + //else + // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); + return count; + } + + private static void WriteBit(byte[] b, bool bit, long index) + => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0)); + //private static void WriteByte(byte[] b, dynamic value, long index) => WriteByte(b, (byte)value, index); + private static void WriteByte(byte[] b, byte value, long index) + { + int byteIndex = (int)(index / 8); + int shift = (int)(index % 8); + byte upper_mask = (byte)(0xFF << shift); + byte lower_mask = (byte)~upper_mask; + + b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift)); + if(shift != 0 && byteIndex + 1 < b.Length) + b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift))); + } + private static void WriteBits(byte[] b, byte value, int bits, int offset, long index) + { + for (int i = 0; i < bits; ++i) + WriteBit(b, (value & (1 << (i + offset))) != 0, index + i); + } + private static void WriteDynamic(byte[] b, int value, int byteCount, long index) + { + for (int i = 0; i < byteCount; ++i) + WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i)); + } + + private static int BytesToRead(object i) + { + bool size; + ulong integer; + if ((size = i is float) || i is double) + { + int bytes = size ? 4 : 8; + Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array + Array result_holder = size ? holder_i as Array : holder_u as Array; + lock (result_holder) + lock (type_holder) + { + // Clear artifacts + if (size) result_holder.SetValue(0U, 0); + else result_holder.SetValue(0UL, 0); + + type_holder.SetValue(i, 0); // Insert the value to convert into the preallocated holder array + Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder + if(size) integer = BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)); + else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)); + } + } + else integer = i as ulong? ?? 0; + return + integer <= 240 ? 1 : + integer <= 2287 ? 2 : + integer <= 67823 ? 3 : + integer <= 16777215 ? 4 : + integer <= 4294967295 ? 5 : + integer <= 1099511627775 ? 6 : + integer <= 281474976710655 ? 7 : + integer <= 72057594037927935 ? 8 : + 9; + } + + // Supported datatypes for serialization + private static bool IsSupportedType(Type t) => supportedTypes.Contains(t); + + // Specifies how many bits will be written + private static long GetBitAllocation(Type t) => + t == typeof(bool) ? 1 : + t == typeof(byte) ? 8 : + t == typeof(sbyte) ? 8 : + t == typeof(short) ? 16 : + t == typeof(char) ? 16 : + t == typeof(ushort) ? 16 : + t == typeof(int) ? 32 : + t == typeof(uint) ? 32 : + t == typeof(long) ? 64 : + t == typeof(ulong) ? 64 : + t == typeof(float) ? 32 : + t == typeof(double) ? 64 : + t == typeof(decimal) ? 128 : + 0; // Unknown type + + // Creates a weak reference to the allocated collector so that reuse may be possible + public void Dispose() + { + expired.Add(new WeakReference(collect)); + collect = null; + } + } +} diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs new file mode 100644 index 0000000..0ee8fc7 --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs @@ -0,0 +1,106 @@ +using MLAPI.NetworkingManagerComponents.Binary; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace Tofvesson.Common +{ + public class BinaryDistributor + { + private static readonly float[] holder_f = new float[1]; + private static readonly double[] holder_d = new double[1]; + private static readonly ulong[] holder_u = new ulong[1]; + private static readonly uint[] holder_i = new uint[1]; + + private readonly byte[] readFrom; + private long bitCount = 0; + public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom; + + public bool ReadBit() + { + bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0; + ++bitCount; + return result; + } + + public byte ReadByte() + { + int shift = (int)(bitCount % 8); + int index = (int)(bitCount / 8); + byte lower_mask = (byte)(0xFF << shift); + byte upper_mask = (byte)~lower_mask; + byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift))); + bitCount += 8; + return result; + } + + public float ReadFloat() => ReadFloating(); + public double ReadDouble() => ReadFloating(); + public float[] ReadFloatArray() => ReadFloatingArray(); + public double[] ReadDoubleArray() => ReadFloatingArray(); + public ushort ReadUShort() => ReadUnsigned(); + public uint ReadUInt() => ReadUnsigned(); + public ulong ReadULong() => ReadUnsigned(); + public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1); + public short ReadShort() => (short)ZigZagDecode(ReadUShort(), 2); + public int ReadInt() => (int)ZigZagDecode(ReadUInt(), 4); + public long ReadLong() => (long)ZigZagDecode(ReadULong(), 8); + + private T ReadUnsigned() + { + dynamic header = ReadByte(); + if (header <= 240) return (T) header; + if (header <= 248) return (T) (240 + 256 * (header - 241) + ReadByte()); + if (header == 249) return (T) (header = 2288 + 256 * ReadByte() + ReadByte()); + dynamic res = ReadByte() | ((long)ReadByte() << 8) | ((long)ReadByte() << 16); + if(header > 250) + { + res |= (long) ReadByte() << 24; + if(header > 251) + { + res |= (long)ReadByte() << 32; + if(header > 252) + { + res |= (long)ReadByte() << 40; + if (header > 253) + { + res |= (long)ReadByte() << 48; + if (header > 254) res |= (long)ReadByte() << 56; + } + } + } + } + return (T) res; + } + private T[] ReadFloatingArray() + { + ushort size = ReadUShort(); + T[] result = new T[size]; + for (short s = 0; s < size; ++s) + result[s] = ReadFloating(); + return result; + } + + private T ReadFloating() + { + int size = Marshal.SizeOf(typeof(T)); + Array type_holder = size == 4 ? holder_f as Array : holder_d as Array; + Array result_holder = size == 4 ? holder_i as Array : holder_u as Array; + T result; + lock(result_holder) + lock (type_holder) + { + //for (int i = 0; i < size; ++i) + // holder.SetValue(ReadByte(), i); + if (size == 4) result_holder.SetValue(BinaryHelpers.SwapEndian(ReadUInt()), 0); + else result_holder.SetValue(BinaryHelpers.SwapEndian(ReadULong()), 0); + Buffer.BlockCopy(result_holder, 0, type_holder, 0, size); + result = (T)type_holder.GetValue(0); + } + return result; + } + private static long ZigZagDecode(ulong d, int bytes) => (long)(((d << (bytes * 8 - 1)) & 1) | (d >> 1)); + } +} diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs new file mode 100644 index 0000000..f9de861 --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace MLAPI.NetworkingManagerComponents.Binary +{ + public static class BinaryHelpers + { + // Swap endianness of a given integer + public static uint SwapEndian(uint value) => (uint)(((value >> 24) & (255 << 0)) | ((value >> 8) & (255 << 8)) | ((value << 8) & (255 << 16)) | ((value << 24) & (255 << 24))); + public static ulong SwapEndian(ulong value) => + ((value >> 56) & 0xFF) | + ((value >> 40) & (0xFFUL << 8)) | + ((value >> 24) & (0xFFUL << 16)) | + ((value >> 8) & (0xFFUL << 24)) | + ((value << 56) & (0xFFUL << 56)) | + ((value << 40) & (0xFFUL << 48)) | + ((value << 24) & (0xFFUL << 40)) | + ((value << 8) & (0xFFUL << 32)) ; + } +} From 06fdaa97d893c84486266e3b8e7cbc33640804e8 Mon Sep 17 00:00:00 2001 From: GabrielTofvesson Date: Tue, 10 Apr 2018 22:23:06 +0200 Subject: [PATCH 03/10] Completely removed dynamic Made specific setters and getters for serialization and deserialization respectively --- .../Binary/BinaryCollector.cs | 41 ++++++++++--- .../Binary/BinaryDistributor.cs | 61 +++++++++++-------- 2 files changed, 68 insertions(+), 34 deletions(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs index 80cb2f6..ae47506 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs @@ -74,7 +74,7 @@ namespace Tofvesson.Common collect = new object[bufferSize]; } - public void Push(T b) + private void Push(T b) { if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) collect[collectCount++] = b is string ? Encoding.UTF8.GetBytes(b as string) : b as object; @@ -82,6 +82,30 @@ namespace Tofvesson.Common // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); } + + public void WriteBool(bool b) => Push(b); + public void WriteFloat(float f) => Push(f); + public void WriteDouble(double d) => Push(d); + public void WriteByte(byte b) => Push(b); + public void WriteUShort(ushort s) => Push(s); + public void WriteUInt(uint i) => Push(i); + public void WriteULong(ulong l) => Push(l); + public void WriteSByte(sbyte b) => Push(b); + public void WriteShort(short s) => Push(s); + public void WriteInt(int i) => Push(i); + public void WriteLong(long l) => Push(l); + public void WriteFloatArray(float[] f) => Push(f); + public void WriteDoubleArray(double[] d) => Push(d); + public void WriteByteArray(byte[] b) => Push(b); + public void WriteUShortArray(ushort[] s) => Push(s); + public void WriteUIntArray(uint[] i) => Push(i); + public void WriteULongArray(ulong[] l) => Push(l); + public void WriteSByteArray(sbyte[] b) => Push(b); + public void WriteShortArray(short[] s) => Push(s); + public void WriteIntArray(int[] i) => Push(i); + public void WriteLongArray(long[] l) => Push(l); + public void WriteString(string s) => Push(s); + public byte[] ToArray() { long bitCount = 0; @@ -135,18 +159,18 @@ namespace Tofvesson.Common else result_holder.SetValue(0UL, 0); type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder - dynamic d = result_holder.GetValue(0); // Since floating point flag bits are seemingly the highest bytes of the floating point values // and even very small values have them, we swap the endianness in the hopes of reducing the size - Serialize(BinaryHelpers.SwapEndian(d), writeTo, ref bitOffset); + if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset); + else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset); } //bitOffset += offset; } else { bool signed = IsSigned(t.GetType()); - dynamic value; + ulong value; if (signed) { Type t1 = t.GetType(); @@ -155,9 +179,12 @@ namespace Tofvesson.Common else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4); else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8); } - else value = t; + else if (t is byte) value = t as byte? ?? 0; + else if (t is ushort) value = t as ushort? ?? 0; + else if (t is uint) value = t as uint? ?? 0; + else /*if (t is ulong)*/ value = t as ulong? ?? 0; - if (value <= 240) WriteByte(writeTo, value, bitOffset); + if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset); else if (value <= 2287) { WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset); @@ -246,7 +273,7 @@ namespace Tofvesson.Common private static void WriteBit(byte[] b, bool bit, long index) => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0)); - //private static void WriteByte(byte[] b, dynamic value, long index) => WriteByte(b, (byte)value, index); + private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index); private static void WriteByte(byte[] b, byte value, long index) { int byteIndex = (int)(index / 8); diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs index 0ee8fc7..fa4857f 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs @@ -9,6 +9,7 @@ namespace Tofvesson.Common { public class BinaryDistributor { + private delegate T Getter(); private static readonly float[] holder_f = new float[1]; private static readonly double[] holder_d = new double[1]; private static readonly ulong[] holder_u = new ulong[1]; @@ -18,13 +19,15 @@ namespace Tofvesson.Common private long bitCount = 0; public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom; - public bool ReadBit() + public bool ReadBool() { bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0; ++bitCount; return result; } + public float ReadFloat() => ReadFloating(); + public double ReadDouble() => ReadFloating(); public byte ReadByte() { int shift = (int)(bitCount % 8); @@ -35,51 +38,55 @@ namespace Tofvesson.Common bitCount += 8; return result; } - - public float ReadFloat() => ReadFloating(); - public double ReadDouble() => ReadFloating(); - public float[] ReadFloatArray() => ReadFloatingArray(); - public double[] ReadDoubleArray() => ReadFloatingArray(); - public ushort ReadUShort() => ReadUnsigned(); - public uint ReadUInt() => ReadUnsigned(); - public ulong ReadULong() => ReadUnsigned(); + public ushort ReadUShort() => (ushort)ReadULong(); + public uint ReadUInt() => (uint)ReadULong(); public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1); public short ReadShort() => (short)ZigZagDecode(ReadUShort(), 2); public int ReadInt() => (int)ZigZagDecode(ReadUInt(), 4); - public long ReadLong() => (long)ZigZagDecode(ReadULong(), 8); + public long ReadLong() => ZigZagDecode(ReadULong(), 8); + public float[] ReadFloatArray() => ReadArray(ReadFloat); + public double[] ReadDoubleArray() => ReadArray(ReadDouble); + public byte[] ReadByteArray() => ReadArray(ReadByte); + public ushort[] ReadUShortArray() => ReadArray(ReadUShort); + public uint[] ReadUIntArray() => ReadArray(ReadUInt); + public ulong[] ReadULongArray() => ReadArray(ReadULong); + public sbyte[] ReadSByteArray() => ReadArray(ReadSByte); + public short[] ReadShortArray() => ReadArray(ReadShort); + public int[] ReadIntArray() => ReadArray(ReadInt); + public long[] ReadLongArray() => ReadArray(ReadLong); + public string ReadString() => Encoding.UTF8.GetString(ReadByteArray()); - private T ReadUnsigned() + private ulong ReadULong() { - dynamic header = ReadByte(); - if (header <= 240) return (T) header; - if (header <= 248) return (T) (240 + 256 * (header - 241) + ReadByte()); - if (header == 249) return (T) (header = 2288 + 256 * ReadByte() + ReadByte()); - dynamic res = ReadByte() | ((long)ReadByte() << 8) | ((long)ReadByte() << 16); + ulong header = ReadByte(); + if (header <= 240) return header; + if (header <= 248) return 240 + 256 * (header - 241) + ReadByte(); + if (header == 249) return 2288 + 256UL * ReadByte() + ReadByte(); + ulong res = ReadByte() | ((ulong)ReadByte() << 8) | ((ulong)ReadByte() << 16); if(header > 250) { - res |= (long) ReadByte() << 24; + res |= (ulong) ReadByte() << 24; if(header > 251) { - res |= (long)ReadByte() << 32; + res |= (ulong)ReadByte() << 32; if(header > 252) { - res |= (long)ReadByte() << 40; + res |= (ulong)ReadByte() << 40; if (header > 253) { - res |= (long)ReadByte() << 48; - if (header > 254) res |= (long)ReadByte() << 56; + res |= (ulong)ReadByte() << 48; + if (header > 254) res |= (ulong)ReadByte() << 56; } } } } - return (T) res; + return res; } - private T[] ReadFloatingArray() + private T[] ReadArray(Getter g) { - ushort size = ReadUShort(); - T[] result = new T[size]; - for (short s = 0; s < size; ++s) - result[s] = ReadFloating(); + T[] result = new T[ReadUShort()]; + for (ushort s = 0; s < result.Length; ++s) + result[s] = g(); return result; } From 18d965d562022c690620aa2b478a4498c6bbd6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Wed, 11 Apr 2018 09:43:45 +0100 Subject: [PATCH 04/10] Changed BitWriter class to take a pre allocated buffer --- MLAPI/MLAPI.csproj | 4 +- .../{BinaryDistributor.cs => BitReader.cs} | 11 ++---- .../{BinaryCollector.cs => BitWriter.cs} | 39 ++++++++++++------- 3 files changed, 32 insertions(+), 22 deletions(-) rename MLAPI/NetworkingManagerComponents/Binary/{BinaryDistributor.cs => BitReader.cs} (94%) rename MLAPI/NetworkingManagerComponents/Binary/{BinaryCollector.cs => BitWriter.cs} (94%) diff --git a/MLAPI/MLAPI.csproj b/MLAPI/MLAPI.csproj index 9c7949f..9b8a4bc 100644 --- a/MLAPI/MLAPI.csproj +++ b/MLAPI/MLAPI.csproj @@ -78,8 +78,8 @@ - - + + diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs similarity index 94% rename from MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs rename to MLAPI/NetworkingManagerComponents/Binary/BitReader.cs index fa4857f..2541fc4 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BinaryDistributor.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs @@ -1,13 +1,10 @@ -using MLAPI.NetworkingManagerComponents.Binary; -using System; -using System.Collections.Generic; -using System.Linq; +using System; using System.Runtime.InteropServices; using System.Text; -namespace Tofvesson.Common +namespace MLAPI.NetworkingManagerComponents.Binary { - public class BinaryDistributor + public class BitReader { private delegate T Getter(); private static readonly float[] holder_f = new float[1]; @@ -17,7 +14,7 @@ namespace Tofvesson.Common private readonly byte[] readFrom; private long bitCount = 0; - public BinaryDistributor(byte[] readFrom) => this.readFrom = readFrom; + public BitReader(byte[] readFrom) => this.readFrom = readFrom; public bool ReadBool() { diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs similarity index 94% rename from MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs rename to MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index ae47506..400de4a 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -1,15 +1,12 @@ -using MLAPI.NetworkingManagerComponents.Binary; -using System; +using System; using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Reflection; -using System.Runtime.InteropServices; using System.Text; +using UnityEngine; -namespace Tofvesson.Common +namespace MLAPI.NetworkingManagerComponents.Binary { - public sealed class BinaryCollector : IDisposable + public sealed class BitWiter : IDisposable { // Collects reusable private static readonly List expired = new List(); @@ -41,7 +38,7 @@ namespace Tofvesson.Common dec_hi, dec_flags; - static BinaryCollector() + static BitWiter() { dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic); dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic); @@ -56,7 +53,7 @@ namespace Tofvesson.Common /// /// Allocates a new binary collector. /// - public BinaryCollector(int bufferSize) + public BitWiter(int bufferSize) { this.bufferSize = bufferSize; for (int i = expired.Count - 1; i >= 0; --i) @@ -106,17 +103,33 @@ namespace Tofvesson.Common public void WriteLongArray(long[] l) => Push(l); public void WriteString(string s) => Push(s); - public byte[] ToArray() + public long Finalize(ref byte[] buffer) { + if(buffer == null) + { + Debug.LogWarning("MLAPI: no buffer provided"); + return 0; + } long bitCount = 0; for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); - byte[] alloc = new byte[(bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)]; + if(buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))) + { + Debug.LogWarning("MLAPI: The buffer size is not large enough"); + return 0; + } long bitOffset = 0; foreach (var item in collect) - Serialize(item, alloc, ref bitOffset); + Serialize(item, buffer, ref bitOffset); - return alloc; + return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); + } + + public long GetFinalizeSize() + { + long bitCount = 0; + for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); + return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); } private static void Serialize(T t, byte[] writeTo, ref long bitOffset) From 45f82180d2c9aa6f69d4b473109f0da289b8057c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Wed, 11 Apr 2018 19:03:14 +0100 Subject: [PATCH 05/10] Removed collect list from GC --- .../Binary/BitWriter.cs | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index 400de4a..6f70667 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -6,10 +6,9 @@ using UnityEngine; namespace MLAPI.NetworkingManagerComponents.Binary { - public sealed class BitWiter : IDisposable + public sealed class BitWriter : IDisposable { - // Collects reusable - private static readonly List expired = new List(); + private static readonly Queue> listPool = new Queue>(); private static readonly float[] holder_f = new float[1]; private static readonly double[] holder_d = new double[1]; @@ -38,43 +37,45 @@ namespace MLAPI.NetworkingManagerComponents.Binary dec_hi, dec_flags; - static BitWiter() + static BitWriter() { dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic); dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic); dec_hi = typeof(decimal).GetField("hi", BindingFlags.NonPublic); dec_flags = typeof(decimal).GetField("flags", BindingFlags.NonPublic); + + for (int i = 0; i < 10; i++) + { + listPool.Enqueue(new List()); + } } - private object[] collect; + private List collect = null; + private bool tempAlloc = false; private readonly int bufferSize; private int collectCount = 0; /// /// Allocates a new binary collector. /// - public BitWiter(int bufferSize) + public BitWriter() { - this.bufferSize = bufferSize; - for (int i = expired.Count - 1; i >= 0; --i) - if (expired[i].IsAlive) - { - collect = (object[])expired[i].Target; - if (collect.Length >= bufferSize) - { - expired.RemoveAt(i); // This entry he been un-expired for now - break; - } - } - else expired.RemoveAt(i); // Entry has been collected by GC - if (collect == null || collect.Length < bufferSize) - collect = new object[bufferSize]; + if (listPool.Count == 0) + { + Debug.LogWarning("MLAPI: There can be no more than 10 BitWriters. Have you forgotten do dispose? (It will still work with worse performance)"); + collect = new List(); + tempAlloc = true; + } + else + { + collect = listPool.Dequeue(); + } } private void Push(T b) { if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) - collect[collectCount++] = b is string ? Encoding.UTF8.GetBytes(b as string) : b as object; + collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object); //else // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); } @@ -122,7 +123,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary foreach (var item in collect) Serialize(item, buffer, ref bitOffset); - return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); + return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1); } public long GetFinalizeSize() @@ -367,8 +368,12 @@ namespace MLAPI.NetworkingManagerComponents.Binary // Creates a weak reference to the allocated collector so that reuse may be possible public void Dispose() { - expired.Add(new WeakReference(collect)); - collect = null; + if (!tempAlloc) + { + collect.Clear(); + listPool.Enqueue(collect); + } + collect = null; //GC picks this } } } From b124f5ed3ee5a4d11c50c6db94d8861cdbe89608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Wed, 11 Apr 2018 19:03:50 +0100 Subject: [PATCH 06/10] Removed unused bufferSize --- MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index 6f70667..68118ff 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -52,7 +52,6 @@ namespace MLAPI.NetworkingManagerComponents.Binary private List collect = null; private bool tempAlloc = false; - private readonly int bufferSize; private int collectCount = 0; /// From 902fb77ed4ba38306fb82f67ccde8b35cb0659b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Thu, 12 Apr 2018 16:12:57 +0100 Subject: [PATCH 07/10] Added support for fixed size arrays --- .../Binary/BitReader.cs | 24 ++++++++-------- .../Binary/BitWriter.cs | 28 +++++++++++-------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs index 2541fc4..af448d5 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs @@ -41,16 +41,16 @@ namespace MLAPI.NetworkingManagerComponents.Binary public short ReadShort() => (short)ZigZagDecode(ReadUShort(), 2); public int ReadInt() => (int)ZigZagDecode(ReadUInt(), 4); public long ReadLong() => ZigZagDecode(ReadULong(), 8); - public float[] ReadFloatArray() => ReadArray(ReadFloat); - public double[] ReadDoubleArray() => ReadArray(ReadDouble); - public byte[] ReadByteArray() => ReadArray(ReadByte); - public ushort[] ReadUShortArray() => ReadArray(ReadUShort); - public uint[] ReadUIntArray() => ReadArray(ReadUInt); - public ulong[] ReadULongArray() => ReadArray(ReadULong); - public sbyte[] ReadSByteArray() => ReadArray(ReadSByte); - public short[] ReadShortArray() => ReadArray(ReadShort); - public int[] ReadIntArray() => ReadArray(ReadInt); - public long[] ReadLongArray() => ReadArray(ReadLong); + public float[] ReadFloatArray(int known = -1) => ReadArray(ReadFloat, known); + public double[] ReadDoubleArray(int known = -1) => ReadArray(ReadDouble, known); + public byte[] ReadByteArray(int known = -1) => ReadArray(ReadByte, known); + public ushort[] ReadUShortArray(int known = -1) => ReadArray(ReadUShort, known); + public uint[] ReadUIntArray(int known = -1) => ReadArray(ReadUInt, known); + public ulong[] ReadULongArray(int known = -1) => ReadArray(ReadULong, known); + public sbyte[] ReadSByteArray(int known = -1) => ReadArray(ReadSByte, known); + public short[] ReadShortArray(int known = -1) => ReadArray(ReadShort, known); + public int[] ReadIntArray(int known = -1) => ReadArray(ReadInt, known); + public long[] ReadLongArray(int known = -1) => ReadArray(ReadLong, known); public string ReadString() => Encoding.UTF8.GetString(ReadByteArray()); private ulong ReadULong() @@ -79,9 +79,9 @@ namespace MLAPI.NetworkingManagerComponents.Binary } return res; } - private T[] ReadArray(Getter g) + private T[] ReadArray(Getter g, int knownSize = -1) { - T[] result = new T[ReadUShort()]; + T[] result = new T[knownSize > 0 ? (uint)knownSize : ReadUInt()]; for (ushort s = 0; s < result.Length; ++s) result[s] = g(); return result; diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index 68118ff..114add6 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -91,17 +91,23 @@ namespace MLAPI.NetworkingManagerComponents.Binary public void WriteShort(short s) => Push(s); public void WriteInt(int i) => Push(i); public void WriteLong(long l) => Push(l); - public void WriteFloatArray(float[] f) => Push(f); - public void WriteDoubleArray(double[] d) => Push(d); - public void WriteByteArray(byte[] b) => Push(b); - public void WriteUShortArray(ushort[] s) => Push(s); - public void WriteUIntArray(uint[] i) => Push(i); - public void WriteULongArray(ulong[] l) => Push(l); - public void WriteSByteArray(sbyte[] b) => Push(b); - public void WriteShortArray(short[] s) => Push(s); - public void WriteIntArray(int[] i) => Push(i); - public void WriteLongArray(long[] l) => Push(l); public void WriteString(string s) => Push(s); + public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known); + public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known); + public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known); + public void WriteUShortArray(ushort[] s, bool known = false) => PushArray(s, known); + public void WriteUIntArray(uint[] i, bool known = false) => PushArray(i, known); + public void WriteULongArray(ulong[] l, bool known = false) => PushArray(l, known); + public void WriteSByteArray(sbyte[] b, bool known = false) => PushArray(b, known); + public void WriteShortArray(short[] s, bool known = false) => PushArray(s, known); + public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known); + public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known); + + private void PushArray(T[] t, bool knownSize = false) + { + if (!knownSize) Push(t); + else foreach (T t1 in t) Push(t1); + } public long Finalize(ref byte[] buffer) { @@ -139,7 +145,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary if (type.IsArray) { var array = t as Array; - Serialize((ushort)array.Length, writeTo, ref bitOffset); + Serialize((uint)array.Length, writeTo, ref bitOffset); foreach (var element in array) Serialize(element, writeTo, ref bitOffset); } From 435db1f14bfe8902c853dfbfb8d656218a332414 Mon Sep 17 00:00:00 2001 From: GabrielTofvesson Date: Fri, 13 Apr 2018 16:28:30 +0200 Subject: [PATCH 08/10] Optimized byte serialization Fixed various bugs Added option to pad misalignments for improved performance (at the possible cost of compression size if called in more than one non-consecutive instance) --- .../Binary/BitReader.cs | 1 + .../Binary/BitWriter.cs | 179 +++++++++--------- 2 files changed, 91 insertions(+), 89 deletions(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs index af448d5..a853b85 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs @@ -35,6 +35,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary bitCount += 8; return result; } + public void SkipPadded() => bitCount += (8 - (bitCount % 8)) % 8; public ushort ReadUShort() => (ushort)ReadULong(); public uint ReadUInt() => (uint)ReadULong(); public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1); diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index 114add6..c5c41f2 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Reflection; +using System.Runtime.InteropServices; using System.Text; using UnityEngine; @@ -52,7 +54,6 @@ namespace MLAPI.NetworkingManagerComponents.Binary private List collect = null; private bool tempAlloc = false; - private int collectCount = 0; /// /// Allocates a new binary collector. @@ -73,10 +74,11 @@ namespace MLAPI.NetworkingManagerComponents.Binary private void Push(T b) { - if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) + if (b == null) collect.Add(b); + else if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object); - //else - // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); + else + Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); } @@ -87,11 +89,12 @@ namespace MLAPI.NetworkingManagerComponents.Binary public void WriteUShort(ushort s) => Push(s); public void WriteUInt(uint i) => Push(i); public void WriteULong(ulong l) => Push(l); - public void WriteSByte(sbyte b) => Push(b); - public void WriteShort(short s) => Push(s); - public void WriteInt(int i) => Push(i); - public void WriteLong(long l) => Push(l); + public void WriteSByte(sbyte b) => Push(ZigZagEncode(b, 8)); + public void WriteShort(short s) => Push(ZigZagEncode(s, 8)); + public void WriteInt(int i) => Push(ZigZagEncode(i, 8)); + public void WriteLong(long l) => Push(ZigZagEncode(l, 8)); public void WriteString(string s) => Push(s); + public void WriteAlignBits() => Push(null); public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known); public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known); public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known); @@ -103,10 +106,12 @@ namespace MLAPI.NetworkingManagerComponents.Binary public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known); public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known); - private void PushArray(T[] t, bool knownSize = false) + public void PushArray(T[] t, bool knownSize = false) { - if (!knownSize) Push(t); - else foreach (T t1 in t) Push(t1); + if (!knownSize) Push((uint)t.Length); + bool signed = IsSigned(t.GetType().GetElementType()); + int size = Marshal.SizeOf(t.GetType().GetElementType()); + foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1); } public long Finalize(ref byte[] buffer) @@ -117,16 +122,22 @@ namespace MLAPI.NetworkingManagerComponents.Binary return 0; } long bitCount = 0; - for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); - if(buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))) + if (buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))) { Debug.LogWarning("MLAPI: The buffer size is not large enough"); return 0; } long bitOffset = 0; + bool isAligned = true; foreach (var item in collect) - Serialize(item, buffer, ref bitOffset); + if (item == null) + { + bitOffset += (8 - (bitOffset % 8)) % 8; + isAligned = true; + } + else Serialize(item, buffer, ref bitOffset, ref isAligned); return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1); } @@ -134,35 +145,36 @@ namespace MLAPI.NetworkingManagerComponents.Binary public long GetFinalizeSize() { long bitCount = 0; - for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]); + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i]==null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); } - private static void Serialize(T t, byte[] writeTo, ref long bitOffset) + private static void Serialize(T t, byte[] writeTo, ref long bitOffset, ref bool isAligned) { Type type = t.GetType(); bool size = false; if (type.IsArray) { var array = t as Array; - Serialize((uint)array.Length, writeTo, ref bitOffset); + Serialize((uint)array.Length, writeTo, ref bitOffset, ref isAligned); foreach (var element in array) - Serialize(element, writeTo, ref bitOffset); + Serialize(element, writeTo, ref bitOffset, ref isAligned); } else if (IsSupportedType(type)) { - long offset = GetBitAllocation(type); + long offset = t is bool ? 1 : BytesToRead(t) * 8; if (type == typeof(bool)) { WriteBit(writeTo, t as bool? ?? false, bitOffset); bitOffset += offset; + isAligned = bitOffset % 8 == 0; } else if (type == typeof(decimal)) { - WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset); - WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32); - WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64); - WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96); + WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset, isAligned); + WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32, isAligned); + WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64, isAligned); + WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96, isAligned); bitOffset += offset; } else if ((size = type == typeof(float)) || type == typeof(double)) @@ -181,71 +193,75 @@ namespace MLAPI.NetworkingManagerComponents.Binary // Since floating point flag bits are seemingly the highest bytes of the floating point values // and even very small values have them, we swap the endianness in the hopes of reducing the size - if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset); - else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset); + if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned); + else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned); } - //bitOffset += offset; } else { - bool signed = IsSigned(t.GetType()); + //bool signed = IsSigned(t.GetType()); ulong value; - if (signed) + /*if (signed) { Type t1 = t.GetType(); if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1); else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2); else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4); - else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8); + else /*if (t1 == typeof(long)) value = (ulong)ZigZagEncode(t as long? ?? 0, 8); + } + else*/ + if (t is byte) + { + WriteByte(writeTo, t as byte? ?? 0, bitOffset, isAligned); + return; } - else if (t is byte) value = t as byte? ?? 0; else if (t is ushort) value = t as ushort? ?? 0; else if (t is uint) value = t as uint? ?? 0; else /*if (t is ulong)*/ value = t as ulong? ?? 0; - if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset); + if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset, isAligned); else if (value <= 2287) { - WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset); - WriteByte(writeTo, (value - 240) % 256, bitOffset + 8); + WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset, isAligned); + WriteByte(writeTo, (value - 240) % 256, bitOffset + 8, isAligned); } else if (value <= 67823) { - WriteByte(writeTo, 249, bitOffset); - WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8); - WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16); + WriteByte(writeTo, 249, bitOffset, isAligned); + WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8, isAligned); + WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16, isAligned); } else { - WriteByte(writeTo, value & 255, bitOffset + 8); - WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16); - WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24); + WriteByte(writeTo, value & 255, bitOffset + 8, isAligned); + WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16, isAligned); + WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24, isAligned); if (value > 16777215) { - WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32); + WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32, isAligned); if (value > 4294967295) { - WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40); + WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40, isAligned); if (value > 1099511627775) { - WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48); + WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48, isAligned); if (value > 281474976710655) { - WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56); + WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56, isAligned); if (value > 72057594037927935) { - WriteByte(writeTo, 255, bitOffset); - WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64); + WriteByte(writeTo, 255, bitOffset, isAligned); + WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64, isAligned); } - else WriteByte(writeTo, 254, bitOffset); + else WriteByte(writeTo, 254, bitOffset, isAligned); } - else WriteByte(writeTo, 253, bitOffset); + else WriteByte(writeTo, 253, bitOffset, isAligned); } - else WriteByte(writeTo, 252, bitOffset); + else WriteByte(writeTo, 252, bitOffset, isAligned); } - else WriteByte(writeTo, 251, bitOffset); + else WriteByte(writeTo, 251, bitOffset, isAligned); } - else WriteByte(writeTo, 250, bitOffset); + else WriteByte(writeTo, 250, bitOffset, isAligned); } bitOffset += BytesToRead(value) * 8; } @@ -255,7 +271,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits)))); private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset)); - private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null)); + private static bool IsSigned(Type t) => t == typeof(sbyte) || t == typeof(short) || t == typeof(int) || t == typeof(long); private static Type GetUnsignedType(Type t) => t == typeof(sbyte) ? typeof(byte) : @@ -274,13 +290,16 @@ namespace MLAPI.NetworkingManagerComponents.Binary { Type elementType = type.GetElementType(); - count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements - foreach (var element in t as Array) - count += GetBitCount(element); + count += BytesToRead((t as Array).Length) * 8; // Int16 array size. Arrays shouldn't be syncing more than 65k elements + + if (elementType == typeof(bool)) count += (t as Array).Length; + else + foreach (var element in t as Array) + count += GetBitCount(element); } else if (IsSupportedType(type)) { - long ba = GetBitAllocation(type); + long ba = t is bool ? 1 : BytesToRead(t)*8; if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string); else if (t is bool || t is decimal) count += ba; else count += BytesToRead(t) * 8; @@ -292,33 +311,32 @@ namespace MLAPI.NetworkingManagerComponents.Binary private static void WriteBit(byte[] b, bool bit, long index) => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0)); - private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index); - private static void WriteByte(byte[] b, byte value, long index) + private static void WriteByte(byte[] b, ulong value, long index, bool isAligned) => WriteByte(b, (byte)value, index, isAligned); + private static void WriteByte(byte[] b, byte value, long index, bool isAligned) { - int byteIndex = (int)(index / 8); - int shift = (int)(index % 8); - byte upper_mask = (byte)(0xFF << shift); - byte lower_mask = (byte)~upper_mask; + if (isAligned) b[index / 8] = value; + else + { + int byteIndex = (int)(index / 8); + int shift = (int)(index % 8); + byte upper_mask = (byte)(0xFF << shift); - b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift)); - if(shift != 0 && byteIndex + 1 < b.Length) + b[byteIndex] = (byte)((b[byteIndex] & (byte)~upper_mask) | (value << shift)); b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift))); + } } - private static void WriteBits(byte[] b, byte value, int bits, int offset, long index) - { - for (int i = 0; i < bits; ++i) - WriteBit(b, (value & (1 << (i + offset))) != 0, index + i); - } - private static void WriteDynamic(byte[] b, int value, int byteCount, long index) + private static void WriteDynamic(byte[] b, int value, int byteCount, long index, bool isAligned) { for (int i = 0; i < byteCount; ++i) - WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i)); + WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i), isAligned); } private static int BytesToRead(object i) { + if (i is byte) return 1; bool size; ulong integer; + if (i is decimal) return BytesToRead((int)dec_flags.GetValue(i)) + BytesToRead((int)dec_lo.GetValue(i)) + BytesToRead((int)dec_mid.GetValue(i)) + BytesToRead((int)dec_hi.GetValue(i)); if ((size = i is float) || i is double) { int bytes = size ? 4 : 8; @@ -337,7 +355,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)); } } - else integer = i as ulong? ?? 0; + else integer = i as ulong? ?? i as uint? ?? i as ushort? ?? i as byte? ?? 0; return integer <= 240 ? 1 : integer <= 2287 ? 2 : @@ -352,24 +370,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary // Supported datatypes for serialization private static bool IsSupportedType(Type t) => supportedTypes.Contains(t); - - // Specifies how many bits will be written - private static long GetBitAllocation(Type t) => - t == typeof(bool) ? 1 : - t == typeof(byte) ? 8 : - t == typeof(sbyte) ? 8 : - t == typeof(short) ? 16 : - t == typeof(char) ? 16 : - t == typeof(ushort) ? 16 : - t == typeof(int) ? 32 : - t == typeof(uint) ? 32 : - t == typeof(long) ? 64 : - t == typeof(ulong) ? 64 : - t == typeof(float) ? 32 : - t == typeof(double) ? 64 : - t == typeof(decimal) ? 128 : - 0; // Unknown type - + // Creates a weak reference to the allocated collector so that reuse may be possible public void Dispose() { From a588584673031483dbc979419125c49b5fadcf9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Sat, 14 Apr 2018 09:08:18 +0100 Subject: [PATCH 09/10] Added allocating Finalize overload to BitWriter --- .../Binary/BitWriter.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs index c5c41f2..5a6c46b 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Reflection; using System.Runtime.InteropServices; using System.Text; @@ -114,6 +113,26 @@ namespace MLAPI.NetworkingManagerComponents.Binary foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1); } + public byte[] Finalize() + { + long bitCount = 0; + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); + byte[] buffer = new byte[((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))]; + + long bitOffset = 0; + bool isAligned = true; + foreach (var item in collect) + if (item == null) + { + bitOffset += (8 - (bitOffset % 8)) % 8; + isAligned = true; + } + else Serialize(item, buffer, ref bitOffset, ref isAligned); + + return buffer; + } + + //The ref is not needed. It's purley there to indicate that it's treated as a reference inside the method. public long Finalize(ref byte[] buffer) { if(buffer == null) @@ -145,7 +164,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary public long GetFinalizeSize() { long bitCount = 0; - for (int i = 0; i < collect.Count; ++i) bitCount += collect[i]==null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); } From a94acfe34d8237d276523bd1e49a3efa666ed733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Cor=C3=A9n?= <2108U9@gmail.com> Date: Tue, 17 Apr 2018 18:26:22 +0200 Subject: [PATCH 10/10] Made ReadULong public --- MLAPI/NetworkingManagerComponents/Binary/BitReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs index a853b85..f76dedb 100644 --- a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs +++ b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs @@ -54,7 +54,7 @@ namespace MLAPI.NetworkingManagerComponents.Binary public long[] ReadLongArray(int known = -1) => ReadArray(ReadLong, known); public string ReadString() => Encoding.UTF8.GetString(ReadByteArray()); - private ulong ReadULong() + public ulong ReadULong() { ulong header = ReadByte(); if (header <= 240) return header;