diff --git a/MLAPI/MLAPI.csproj b/MLAPI/MLAPI.csproj index 8507873..8e56c79 100644 --- a/MLAPI/MLAPI.csproj +++ b/MLAPI/MLAPI.csproj @@ -78,6 +78,9 @@ + + + diff --git a/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs b/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs new file mode 100644 index 0000000..f9de861 --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BinaryHelpers.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace MLAPI.NetworkingManagerComponents.Binary +{ + public static class BinaryHelpers + { + // Swap endianness of a given integer + public static uint SwapEndian(uint value) => (uint)(((value >> 24) & (255 << 0)) | ((value >> 8) & (255 << 8)) | ((value << 8) & (255 << 16)) | ((value << 24) & (255 << 24))); + public static ulong SwapEndian(ulong value) => + ((value >> 56) & 0xFF) | + ((value >> 40) & (0xFFUL << 8)) | + ((value >> 24) & (0xFFUL << 16)) | + ((value >> 8) & (0xFFUL << 24)) | + ((value << 56) & (0xFFUL << 56)) | + ((value << 40) & (0xFFUL << 48)) | + ((value << 24) & (0xFFUL << 40)) | + ((value << 8) & (0xFFUL << 32)) ; + } +} diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs new file mode 100644 index 0000000..f76dedb --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BitReader.cs @@ -0,0 +1,111 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace MLAPI.NetworkingManagerComponents.Binary +{ + public class BitReader + { + private delegate T Getter(); + private static readonly float[] holder_f = new float[1]; + private static readonly double[] holder_d = new double[1]; + private static readonly ulong[] holder_u = new ulong[1]; + private static readonly uint[] holder_i = new uint[1]; + + private readonly byte[] readFrom; + private long bitCount = 0; + public BitReader(byte[] readFrom) => this.readFrom = readFrom; + + public bool ReadBool() + { + bool result = (readFrom[bitCount / 8] & (byte)(1 << (int)(bitCount % 8))) != 0; + ++bitCount; + return result; + } + + public float ReadFloat() => ReadFloating(); + public double ReadDouble() => ReadFloating(); + public byte ReadByte() + { + int shift = (int)(bitCount % 8); + int index = (int)(bitCount / 8); + byte lower_mask = (byte)(0xFF << shift); + byte upper_mask = (byte)~lower_mask; + byte result = (byte)(((readFrom[index] & lower_mask) >> shift) | (shift == 0 ? 0 : (readFrom[index + 1] & upper_mask) << (8 - shift))); + bitCount += 8; + return result; + } + public void SkipPadded() => bitCount += (8 - (bitCount % 8)) % 8; + public ushort ReadUShort() => (ushort)ReadULong(); + public uint ReadUInt() => (uint)ReadULong(); + public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1); + public short ReadShort() => (short)ZigZagDecode(ReadUShort(), 2); + public int ReadInt() => (int)ZigZagDecode(ReadUInt(), 4); + public long ReadLong() => ZigZagDecode(ReadULong(), 8); + public float[] ReadFloatArray(int known = -1) => ReadArray(ReadFloat, known); + public double[] ReadDoubleArray(int known = -1) => ReadArray(ReadDouble, known); + public byte[] ReadByteArray(int known = -1) => ReadArray(ReadByte, known); + public ushort[] ReadUShortArray(int known = -1) => ReadArray(ReadUShort, known); + public uint[] ReadUIntArray(int known = -1) => ReadArray(ReadUInt, known); + public ulong[] ReadULongArray(int known = -1) => ReadArray(ReadULong, known); + public sbyte[] ReadSByteArray(int known = -1) => ReadArray(ReadSByte, known); + public short[] ReadShortArray(int known = -1) => ReadArray(ReadShort, known); + public int[] ReadIntArray(int known = -1) => ReadArray(ReadInt, known); + public long[] ReadLongArray(int known = -1) => ReadArray(ReadLong, known); + public string ReadString() => Encoding.UTF8.GetString(ReadByteArray()); + + public ulong ReadULong() + { + ulong header = ReadByte(); + if (header <= 240) return header; + if (header <= 248) return 240 + 256 * (header - 241) + ReadByte(); + if (header == 249) return 2288 + 256UL * ReadByte() + ReadByte(); + ulong res = ReadByte() | ((ulong)ReadByte() << 8) | ((ulong)ReadByte() << 16); + if(header > 250) + { + res |= (ulong) ReadByte() << 24; + if(header > 251) + { + res |= (ulong)ReadByte() << 32; + if(header > 252) + { + res |= (ulong)ReadByte() << 40; + if (header > 253) + { + res |= (ulong)ReadByte() << 48; + if (header > 254) res |= (ulong)ReadByte() << 56; + } + } + } + } + return res; + } + private T[] ReadArray(Getter g, int knownSize = -1) + { + T[] result = new T[knownSize > 0 ? (uint)knownSize : ReadUInt()]; + for (ushort s = 0; s < result.Length; ++s) + result[s] = g(); + return result; + } + + private T ReadFloating() + { + int size = Marshal.SizeOf(typeof(T)); + Array type_holder = size == 4 ? holder_f as Array : holder_d as Array; + Array result_holder = size == 4 ? holder_i as Array : holder_u as Array; + T result; + lock(result_holder) + lock (type_holder) + { + //for (int i = 0; i < size; ++i) + // holder.SetValue(ReadByte(), i); + if (size == 4) result_holder.SetValue(BinaryHelpers.SwapEndian(ReadUInt()), 0); + else result_holder.SetValue(BinaryHelpers.SwapEndian(ReadULong()), 0); + Buffer.BlockCopy(result_holder, 0, type_holder, 0, size); + result = (T)type_holder.GetValue(0); + } + return result; + } + private static long ZigZagDecode(ulong d, int bytes) => (long)(((d << (bytes * 8 - 1)) & 1) | (d >> 1)); + } +} diff --git a/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs new file mode 100644 index 0000000..5a6c46b --- /dev/null +++ b/MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs @@ -0,0 +1,404 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; +using UnityEngine; + +namespace MLAPI.NetworkingManagerComponents.Binary +{ + public sealed class BitWriter : IDisposable + { + private static readonly Queue> listPool = new Queue>(); + + private static readonly float[] holder_f = new float[1]; + private static readonly double[] holder_d = new double[1]; + private static readonly ulong[] holder_u = new ulong[1]; + private static readonly uint[] holder_i = new uint[1]; + private static readonly List supportedTypes = new List() + { + typeof(bool), + typeof(byte), + typeof(sbyte), + typeof(char), + typeof(short), + typeof(ushort), + typeof(int), + typeof(uint), + typeof(long), + typeof(ulong), + typeof(float), + typeof(double), + typeof(decimal) + }; + + private static readonly FieldInfo + dec_lo, + dec_mid, + dec_hi, + dec_flags; + + static BitWriter() + { + dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic); + dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic); + dec_hi = typeof(decimal).GetField("hi", BindingFlags.NonPublic); + dec_flags = typeof(decimal).GetField("flags", BindingFlags.NonPublic); + + for (int i = 0; i < 10; i++) + { + listPool.Enqueue(new List()); + } + } + + private List collect = null; + private bool tempAlloc = false; + + /// + /// Allocates a new binary collector. + /// + public BitWriter() + { + if (listPool.Count == 0) + { + Debug.LogWarning("MLAPI: There can be no more than 10 BitWriters. Have you forgotten do dispose? (It will still work with worse performance)"); + collect = new List(); + tempAlloc = true; + } + else + { + collect = listPool.Dequeue(); + } + } + + private void Push(T b) + { + if (b == null) collect.Add(b); + else if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType())) + collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object); + else + Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); + } + + + public void WriteBool(bool b) => Push(b); + public void WriteFloat(float f) => Push(f); + public void WriteDouble(double d) => Push(d); + public void WriteByte(byte b) => Push(b); + public void WriteUShort(ushort s) => Push(s); + public void WriteUInt(uint i) => Push(i); + public void WriteULong(ulong l) => Push(l); + public void WriteSByte(sbyte b) => Push(ZigZagEncode(b, 8)); + public void WriteShort(short s) => Push(ZigZagEncode(s, 8)); + public void WriteInt(int i) => Push(ZigZagEncode(i, 8)); + public void WriteLong(long l) => Push(ZigZagEncode(l, 8)); + public void WriteString(string s) => Push(s); + public void WriteAlignBits() => Push(null); + public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known); + public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known); + public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known); + public void WriteUShortArray(ushort[] s, bool known = false) => PushArray(s, known); + public void WriteUIntArray(uint[] i, bool known = false) => PushArray(i, known); + public void WriteULongArray(ulong[] l, bool known = false) => PushArray(l, known); + public void WriteSByteArray(sbyte[] b, bool known = false) => PushArray(b, known); + public void WriteShortArray(short[] s, bool known = false) => PushArray(s, known); + public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known); + public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known); + + public void PushArray(T[] t, bool knownSize = false) + { + if (!knownSize) Push((uint)t.Length); + bool signed = IsSigned(t.GetType().GetElementType()); + int size = Marshal.SizeOf(t.GetType().GetElementType()); + foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1); + } + + public byte[] Finalize() + { + long bitCount = 0; + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); + byte[] buffer = new byte[((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))]; + + long bitOffset = 0; + bool isAligned = true; + foreach (var item in collect) + if (item == null) + { + bitOffset += (8 - (bitOffset % 8)) % 8; + isAligned = true; + } + else Serialize(item, buffer, ref bitOffset, ref isAligned); + + return buffer; + } + + //The ref is not needed. It's purley there to indicate that it's treated as a reference inside the method. + public long Finalize(ref byte[] buffer) + { + if(buffer == null) + { + Debug.LogWarning("MLAPI: no buffer provided"); + return 0; + } + long bitCount = 0; + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); + + if (buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1))) + { + Debug.LogWarning("MLAPI: The buffer size is not large enough"); + return 0; + } + long bitOffset = 0; + bool isAligned = true; + foreach (var item in collect) + if (item == null) + { + bitOffset += (8 - (bitOffset % 8)) % 8; + isAligned = true; + } + else Serialize(item, buffer, ref bitOffset, ref isAligned); + + return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1); + } + + public long GetFinalizeSize() + { + long bitCount = 0; + for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]); + return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)); + } + + private static void Serialize(T t, byte[] writeTo, ref long bitOffset, ref bool isAligned) + { + Type type = t.GetType(); + bool size = false; + if (type.IsArray) + { + var array = t as Array; + Serialize((uint)array.Length, writeTo, ref bitOffset, ref isAligned); + foreach (var element in array) + Serialize(element, writeTo, ref bitOffset, ref isAligned); + } + else if (IsSupportedType(type)) + { + long offset = t is bool ? 1 : BytesToRead(t) * 8; + if (type == typeof(bool)) + { + WriteBit(writeTo, t as bool? ?? false, bitOffset); + bitOffset += offset; + isAligned = bitOffset % 8 == 0; + } + else if (type == typeof(decimal)) + { + WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset, isAligned); + WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32, isAligned); + WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64, isAligned); + WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96, isAligned); + bitOffset += offset; + } + else if ((size = type == typeof(float)) || type == typeof(double)) + { + int bytes = size ? 4 : 8; + Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array + Array result_holder = size ? holder_i as Array : holder_u as Array; + lock (result_holder) + lock (type_holder) + { + // Clear artifacts + if (size) result_holder.SetValue(0U, 0); + else result_holder.SetValue(0UL, 0); + type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array + Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder + + // Since floating point flag bits are seemingly the highest bytes of the floating point values + // and even very small values have them, we swap the endianness in the hopes of reducing the size + if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned); + else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned); + } + } + else + { + //bool signed = IsSigned(t.GetType()); + ulong value; + /*if (signed) + { + Type t1 = t.GetType(); + if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1); + else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2); + else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4); + else /*if (t1 == typeof(long)) value = (ulong)ZigZagEncode(t as long? ?? 0, 8); + } + else*/ + if (t is byte) + { + WriteByte(writeTo, t as byte? ?? 0, bitOffset, isAligned); + return; + } + else if (t is ushort) value = t as ushort? ?? 0; + else if (t is uint) value = t as uint? ?? 0; + else /*if (t is ulong)*/ value = t as ulong? ?? 0; + + if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset, isAligned); + else if (value <= 2287) + { + WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset, isAligned); + WriteByte(writeTo, (value - 240) % 256, bitOffset + 8, isAligned); + } + else if (value <= 67823) + { + WriteByte(writeTo, 249, bitOffset, isAligned); + WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8, isAligned); + WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16, isAligned); + } + else + { + WriteByte(writeTo, value & 255, bitOffset + 8, isAligned); + WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16, isAligned); + WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24, isAligned); + if (value > 16777215) + { + WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32, isAligned); + if (value > 4294967295) + { + WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40, isAligned); + if (value > 1099511627775) + { + WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48, isAligned); + if (value > 281474976710655) + { + WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56, isAligned); + if (value > 72057594037927935) + { + WriteByte(writeTo, 255, bitOffset, isAligned); + WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64, isAligned); + } + else WriteByte(writeTo, 254, bitOffset, isAligned); + } + else WriteByte(writeTo, 253, bitOffset, isAligned); + } + else WriteByte(writeTo, 252, bitOffset, isAligned); + } + else WriteByte(writeTo, 251, bitOffset, isAligned); + } + else WriteByte(writeTo, 250, bitOffset, isAligned); + } + bitOffset += BytesToRead(value) * 8; + } + } + } + + private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits)))); + private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset)); + + private static bool IsSigned(Type t) => t == typeof(sbyte) || t == typeof(short) || t == typeof(int) || t == typeof(long); + + private static Type GetUnsignedType(Type t) => + t == typeof(sbyte) ? typeof(byte) : + t == typeof(short) ? typeof(ushort) : + t == typeof(int) ? typeof(uint) : + t == typeof(long) ? typeof(ulong) : + null; + + private static ulong ZigZagEncode(long d, int bytes) => (ulong)(((d >> (bytes * 8 - 1))&1) | (d << 1)); + + private static long GetBitCount(T t) + { + Type type = t.GetType(); + long count = 0; + if (type.IsArray) + { + Type elementType = type.GetElementType(); + + count += BytesToRead((t as Array).Length) * 8; // Int16 array size. Arrays shouldn't be syncing more than 65k elements + + if (elementType == typeof(bool)) count += (t as Array).Length; + else + foreach (var element in t as Array) + count += GetBitCount(element); + } + else if (IsSupportedType(type)) + { + long ba = t is bool ? 1 : BytesToRead(t)*8; + if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string); + else if (t is bool || t is decimal) count += ba; + else count += BytesToRead(t) * 8; + } + //else + // Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored"); + return count; + } + + private static void WriteBit(byte[] b, bool bit, long index) + => b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0)); + private static void WriteByte(byte[] b, ulong value, long index, bool isAligned) => WriteByte(b, (byte)value, index, isAligned); + private static void WriteByte(byte[] b, byte value, long index, bool isAligned) + { + if (isAligned) b[index / 8] = value; + else + { + int byteIndex = (int)(index / 8); + int shift = (int)(index % 8); + byte upper_mask = (byte)(0xFF << shift); + + b[byteIndex] = (byte)((b[byteIndex] & (byte)~upper_mask) | (value << shift)); + b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift))); + } + } + private static void WriteDynamic(byte[] b, int value, int byteCount, long index, bool isAligned) + { + for (int i = 0; i < byteCount; ++i) + WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i), isAligned); + } + + private static int BytesToRead(object i) + { + if (i is byte) return 1; + bool size; + ulong integer; + if (i is decimal) return BytesToRead((int)dec_flags.GetValue(i)) + BytesToRead((int)dec_lo.GetValue(i)) + BytesToRead((int)dec_mid.GetValue(i)) + BytesToRead((int)dec_hi.GetValue(i)); + if ((size = i is float) || i is double) + { + int bytes = size ? 4 : 8; + Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array + Array result_holder = size ? holder_i as Array : holder_u as Array; + lock (result_holder) + lock (type_holder) + { + // Clear artifacts + if (size) result_holder.SetValue(0U, 0); + else result_holder.SetValue(0UL, 0); + + type_holder.SetValue(i, 0); // Insert the value to convert into the preallocated holder array + Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder + if(size) integer = BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)); + else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)); + } + } + else integer = i as ulong? ?? i as uint? ?? i as ushort? ?? i as byte? ?? 0; + return + integer <= 240 ? 1 : + integer <= 2287 ? 2 : + integer <= 67823 ? 3 : + integer <= 16777215 ? 4 : + integer <= 4294967295 ? 5 : + integer <= 1099511627775 ? 6 : + integer <= 281474976710655 ? 7 : + integer <= 72057594037927935 ? 8 : + 9; + } + + // Supported datatypes for serialization + private static bool IsSupportedType(Type t) => supportedTypes.Contains(t); + + // Creates a weak reference to the allocated collector so that reuse may be possible + public void Dispose() + { + if (!tempAlloc) + { + collect.Clear(); + listPool.Enqueue(collect); + } + collect = null; //GC picks this + } + } +}