From 9662d52f90418baf58d70f1350ab49f6c4716d4f Mon Sep 17 00:00:00 2001 From: Tyler Young Date: Fri, 12 Jun 2020 22:09:48 -0400 Subject: [PATCH] Shared String Dictionary ctd. (#1126) --- Robust.Client/GameController.cs | 8 +- Robust.Client/Interfaces/IGameController.cs | 6 +- Robust.Server/BaseServer.cs | 59 +- Robust.Server/Interfaces/IBaseServer.cs | 3 +- Robust.Server/Interfaces/Maps/IMapLoader.cs | 6 +- Robust.Server/Maps/MapLoader.cs | 13 +- .../IRobustMappedStringSerializer.cs | 229 +++ .../Localization/LocalizationManager.cs | 4 +- Robust.Shared/Prototypes/PrototypeManager.cs | 12 +- Robust.Shared/Robust.Shared.csproj | 12 - ...tMappedStringsSerializerClientHandshake.cs | 45 + ...tMappedStringsSerializerServerHandshake.cs | 54 + ...MsgRobustMappedStringsSerializerStrings.cs | 61 + .../RobustMappedStringSerializer.cs | 1319 +++++++++++++++++ .../RobustSerializer.Handshake.cs | 6 +- ...ppedStringSerializer.MsgClientHandshake.cs | 54 - ...ppedStringSerializer.MsgServerHandshake.cs | 65 - ...lizer.MappedStringSerializer.MsgStrings.cs | 72 - ...RobustSerializer.MappedStringSerializer.cs | 1295 ---------------- .../Serialization/RobustSerializer.cs | 17 +- Robust.Shared/Utility/TypeAbbreviation.cs | 4 +- Robust.UnitTesting/GameControllerDummy.cs | 6 +- Robust.UnitTesting/RobustIntegrationTest.cs | 10 +- Robust.UnitTesting/TestLogHandler.cs | 43 + 24 files changed, 1855 insertions(+), 1548 deletions(-) create mode 100644 Robust.Shared/Interfaces/Serialization/IRobustMappedStringSerializer.cs create mode 100644 Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerClientHandshake.cs create mode 100644 Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerServerHandshake.cs create mode 100644 Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerStrings.cs create mode 100644 Robust.Shared/Serialization/RobustMappedStringSerializer.cs delete mode 100644 Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgClientHandshake.cs delete mode 100644 Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgServerHandshake.cs delete mode 100644 Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgStrings.cs delete mode 100644 Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.cs create mode 100644 Robust.UnitTesting/TestLogHandler.cs diff --git a/Robust.Client/GameController.cs b/Robust.Client/GameController.cs index 91f32ab87..c8bfe74b5 100644 --- a/Robust.Client/GameController.cs +++ b/Robust.Client/GameController.cs @@ -80,11 +80,11 @@ namespace Robust.Client _commandLineArgs = args; } - public bool Startup() + public bool Startup(Func? logHandlerFactory = null) { ReadInitialLaunchState(); - SetupLogging(_logManager); + SetupLogging(_logManager, logHandlerFactory ?? (() => new ConsoleLogHandler())); _taskManager.Initialize(); @@ -286,9 +286,9 @@ namespace Robust.Client _modLoader.BroadcastUpdate(ModUpdateLevel.FramePostEngine, frameEventArgs); } - internal static void SetupLogging(ILogManager logManager) + internal static void SetupLogging(ILogManager logManager, Func logHandlerFactory) { - logManager.RootSawmill.AddHandler(new ConsoleLogHandler()); + logManager.RootSawmill.AddHandler(logHandlerFactory()); logManager.GetSawmill("res.typecheck").Level = LogLevel.Info; logManager.GetSawmill("res.tex").Level = LogLevel.Info; diff --git a/Robust.Client/Interfaces/IGameController.cs b/Robust.Client/Interfaces/IGameController.cs index fc2d4d8ce..5388a2d8f 100644 --- a/Robust.Client/Interfaces/IGameController.cs +++ b/Robust.Client/Interfaces/IGameController.cs @@ -1,5 +1,7 @@ -using System.Net; +using System; +using System.Net; using Robust.Client.Input; +using Robust.Shared.Interfaces.Log; using Robust.Shared.Timing; namespace Robust.Client.Interfaces @@ -15,7 +17,7 @@ namespace Robust.Client.Interfaces { void SetCommandLineArgs(CommandLineArgs args); bool LoadConfigAndUserData { get; set; } - bool Startup(); + bool Startup(Func? logHandlerFactory = null); void MainLoop(GameController.DisplayMode mode); void KeyDown(KeyEventArgs keyEvent); void KeyUp(KeyEventArgs keyEvent); diff --git a/Robust.Server/BaseServer.cs b/Robust.Server/BaseServer.cs index c05633c25..db5220a71 100644 --- a/Robust.Server/BaseServer.cs +++ b/Robust.Server/BaseServer.cs @@ -36,6 +36,8 @@ using Robust.Server.ServerStatus; using Robust.Shared; using Robust.Shared.Network.Messages; using Robust.Server.DataMetrics; +using Robust.Server.Interfaces.Maps; +using Robust.Shared.Serialization; using Stopwatch = Robust.Shared.Timing.Stopwatch; namespace Robust.Server @@ -80,7 +82,8 @@ namespace Robust.Server private readonly Stopwatch _uptimeStopwatch = new Stopwatch(); private CommandLineArgs _commandLineArgs = default!; - private FileLogHandler fileLogHandler = default!; + private Func? _logHandlerFactory; + private ILogHandler? _logHandler; private IGameLoop _mainLoop = default!; private TimeSpan _lastTitleUpdate; @@ -103,7 +106,7 @@ namespace Robust.Server Logger.InfoS("srv", "Restarting Server..."); Cleanup(); - Start(); + Start(_logHandlerFactory); } /// @@ -117,8 +120,11 @@ namespace Robust.Server _shutdownReason = reason; _mainLoop.Running = false; - _log.RootSawmill.RemoveHandler(fileLogHandler); - fileLogHandler.Dispose(); + if (_logHandler != null) + { + _log.RootSawmill.RemoveHandler(_logHandler); + (_logHandler as IDisposable)?.Dispose(); + } } public void SetCommandLineArgs(CommandLineArgs args) @@ -127,7 +133,7 @@ namespace Robust.Server } /// - public bool Start() + public bool Start(Func? logHandlerFactory = null) { // Sets up the configMgr // If a config file path was passed, use it literally. @@ -160,24 +166,40 @@ namespace Robust.Server //Sets up Logging + _config.RegisterCVar("log.enabled", true, CVar.ARCHIVE); _config.RegisterCVar("log.path", "logs", CVar.ARCHIVE); _config.RegisterCVar("log.format", "log_%(date)s-T%(time)s.txt", CVar.ARCHIVE); _config.RegisterCVar("log.level", LogLevel.Info, CVar.ARCHIVE); - var logPath = _config.GetCVar("log.path"); - var logFormat = _config.GetCVar("log.format"); - var logFilename = logFormat.Replace("%(date)s", DateTime.Now.ToString("yyyy-MM-dd")) - .Replace("%(time)s", DateTime.Now.ToString("hh-mm-ss")); - var fullPath = Path.Combine(logPath, logFilename); + _logHandlerFactory = logHandlerFactory; - if (!Path.IsPathRooted(fullPath)) + var logHandler = logHandlerFactory?.Invoke() ?? null; + + var logEnabled = _config.GetCVar("log.enabled"); + + if (logEnabled && logHandler == null) { - logPath = PathHelpers.ExecutableRelativeFile(fullPath); + var logPath = _config.GetCVar("log.path"); + var logFormat = _config.GetCVar("log.format"); + var logFilename = logFormat.Replace("%(date)s", DateTime.Now.ToString("yyyy-MM-dd")) + .Replace("%(time)s", DateTime.Now.ToString("hh-mm-ss")); + var fullPath = Path.Combine(logPath, logFilename); + + if (!Path.IsPathRooted(fullPath)) + { + logPath = PathHelpers.ExecutableRelativeFile(fullPath); + } + + logHandler = new FileLogHandler(logPath); } - fileLogHandler = new FileLogHandler(logPath); _log.RootSawmill.Level = _config.GetCVar("log.level"); - _log.RootSawmill.AddHandler(fileLogHandler); + + if (logEnabled && logHandler != null) + { + _logHandler = logHandler; + _log.RootSawmill.AddHandler(_logHandler!); + } // Has to be done early because this guy's in charge of the main thread Synchronization Context. _taskManager.Initialize(); @@ -241,6 +263,11 @@ namespace Robust.Server // TODO: solve this properly. _serializer.Initialize(); + //IoCManager.Resolve().LoadedMapData += + // IoCManager.Resolve().AddStrings; + IoCManager.Resolve().LoadedData += + IoCManager.Resolve().AddStrings; + // Initialize Tier 2 services IoCManager.Resolve().InSimulation = true; @@ -395,7 +422,9 @@ namespace Robust.Server // Wrtie down exception log var logPath = _config.GetCVar("log.path"); - var pathToWrite = Path.Combine(PathHelpers.ExecutableRelativeFile(logPath), + var relPath = PathHelpers.ExecutableRelativeFile(logPath); + Directory.CreateDirectory(relPath); + var pathToWrite = Path.Combine(relPath, "Runtime-" + DateTime.Now.ToString("yyyy-MM-dd-THH-mm-ss") + ".txt"); File.WriteAllText(pathToWrite, runtimeLog.Display(), EncodingHelpers.UTF8); diff --git a/Robust.Server/Interfaces/IBaseServer.cs b/Robust.Server/Interfaces/IBaseServer.cs index f161d561b..351c13d49 100644 --- a/Robust.Server/Interfaces/IBaseServer.cs +++ b/Robust.Server/Interfaces/IBaseServer.cs @@ -1,5 +1,6 @@ using System; using Robust.Server.Player; +using Robust.Shared.Interfaces.Log; using Robust.Shared.Timing; namespace Robust.Server.Interfaces @@ -23,7 +24,7 @@ namespace Robust.Server.Interfaces /// Sets up the server, loads the game, gets ready for client connections. /// /// - bool Start(); + bool Start(Func? logHandler = null); /// /// Hard restarts the server, shutting it down, kicking all players, and starting the server again. diff --git a/Robust.Server/Interfaces/Maps/IMapLoader.cs b/Robust.Server/Interfaces/Maps/IMapLoader.cs index 8013c2b8f..c43d8c1b6 100644 --- a/Robust.Server/Interfaces/Maps/IMapLoader.cs +++ b/Robust.Server/Interfaces/Maps/IMapLoader.cs @@ -1,4 +1,6 @@ -using Robust.Shared.Map; +using System; +using Robust.Shared.Map; +using YamlDotNet.RepresentationModel; namespace Robust.Server.Interfaces.Maps { @@ -9,5 +11,7 @@ namespace Robust.Server.Interfaces.Maps void LoadMap(MapId mapId, string path); void SaveMap(MapId mapId, string yamlPath); + + event Action LoadedMapData; } } diff --git a/Robust.Server/Maps/MapLoader.cs b/Robust.Server/Maps/MapLoader.cs index 4afcffc3b..0490b9459 100644 --- a/Robust.Server/Maps/MapLoader.cs +++ b/Robust.Server/Maps/MapLoader.cs @@ -39,6 +39,8 @@ namespace Robust.Server.Maps [Dependency] private readonly IComponentManager _componentManager = default!; [Dependency] private readonly IPrototypeManager _prototypeManager = default!; + public event Action? LoadedMapData; + /// public void SaveBlueprint(GridId gridId, string yamlPath) { @@ -99,6 +101,8 @@ namespace Robust.Server.Maps var data = new MapData(reader); + LoadedMapData?.Invoke(data.Stream, resPath.ToString()); + if (data.GridCount != 1) { throw new InvalidDataException("Cannot instance map with multiple grids as blueprint."); @@ -182,6 +186,8 @@ namespace Robust.Server.Maps var data = new MapData(reader); + LoadedMapData?.Invoke(data.Stream, resPath.ToString()); + var context = new MapContext(_mapManager, _tileDefinitionManager, _serverEntityManager, _pauseManager, _componentManager, _prototypeManager, (YamlMappingNode)data.RootNode, mapId); context.Deserialize(); @@ -849,7 +855,9 @@ namespace Robust.Server.Maps /// private class MapData { - public YamlNode RootNode { get; } + public YamlStream Stream { get; } + + public YamlNode RootNode => Stream.Documents[0].RootNode; public int GridCount { get; } public MapData(TextReader reader) @@ -869,9 +877,8 @@ namespace Robust.Server.Maps throw new InvalidDataException("Stream too many YAML documents. Map files store exactly one."); } - RootNode = stream.Documents[0].RootNode; + Stream = stream; GridCount = ((YamlSequenceNode)RootNode["grids"]).Children.Count; - RobustSerializer.MappedStringSerializer.AddStrings(stream, "anonymous map YAML stream"); } } } diff --git a/Robust.Shared/Interfaces/Serialization/IRobustMappedStringSerializer.cs b/Robust.Shared/Interfaces/Serialization/IRobustMappedStringSerializer.cs new file mode 100644 index 000000000..ca62dd102 --- /dev/null +++ b/Robust.Shared/Interfaces/Serialization/IRobustMappedStringSerializer.cs @@ -0,0 +1,229 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading.Tasks; +using JetBrains.Annotations; +using NetSerializer; +using Newtonsoft.Json.Linq; +using Robust.Shared.Interfaces.Network; +using YamlDotNet.RepresentationModel; + +namespace Robust.Shared.Serialization +{ + + [PublicAPI] + public interface IRobustMappedStringSerializer + { + + /// + /// Starts the handshake from the server end of the given channel, + /// sending a . + /// + /// The network channel to perform the handshake over. + /// + /// Locks the string mapping if this is the first time the server is + /// performing the handshake. + /// + /// + /// + Task Handshake(INetChannel channel); + + /// + /// Performs the setup so that the serializer can perform the string- + /// exchange protocol. + /// + /// + /// The string-exchange protocol is started by the server when the + /// client first connects. The server sends the client a hash of the + /// string mapping; the client checks that hash against any local + /// caches; and if necessary, the client requests a new copy of the + /// mapping from the server. + /// + /// Uncached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// Cached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Dont Need Strings -> | + /// + /// + /// Verification failure flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// + Hash Failed | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// NOTE: Verification failure flow is currently not implemented. + /// + /// + /// The to perform the protocol steps over. + /// + /// + /// + /// + /// + /// + /// + /// + void NetworkInitialize(INetManager net); + + /// + /// Writes a strings package to a stream. + /// + /// A writable stream. + /// Overly long string in strings package. + void WriteStringPackage(Stream stream); + + /// + /// Converts a URL-safe Base64 string into a byte array. + /// + /// A base64url formed string. + /// The represented byte array. + byte[] ConvertFromBase64Url(string s); + + IReadOnlyList MappedStrings { get; } + + /// + /// Whether the string mapping is decided, and cannot be changed. + /// + /// + /// + /// While false, strings can be added to the mapping, but + /// it cannot be saved to a cache. + /// + /// + /// While true, the mapping cannot be modified, but can be + /// shared between the server and client and saved to a cache. + /// + /// + bool LockMappedStrings { get; set; } + + /// + /// The hash of the string mapping. + /// + /// + /// Thrown if the mapping is not locked. + /// + byte[] MappedStringsHash { get; } + + /// + /// Add a string to the constant mapping. + /// + /// + /// If the string has multiple detectable subcomponents, such as a + /// filepath, it may result in more than one string being added to + /// the mapping. As string parts are commonly sent as subsets or + /// scoped names, this increases the likelyhood of a successful + /// string mapping. + /// + /// + /// true if the string was added to the mapping for the first + /// time, false otherwise. + /// + /// + /// Thrown if the string is not normalized (). + /// + bool AddString(string str); + + /// + /// Add the constant strings from an to the + /// mapping. + /// + /// The assembly from which to collect constant strings. + void AddStrings(Assembly asm); + + /// + /// Add strings from the given to the mapping. + /// + /// + /// Strings are taken from YAML anchors, tags, and leaf nodes. + /// + /// The YAML to collect strings from. + /// The stream name. Only used for logging. + void AddStrings(YamlStream yaml, string name); + + /// + /// Add strings from the given to the mapping. + /// + /// + /// Strings are taken from JSON property names and string nodes. + /// + /// The JSON to collect strings from. + /// The stream name. Only used for logging. + void AddStrings(JObject obj, string name); + + /// + /// Remove all strings from the mapping, completely resetting it. + /// + /// + /// Thrown if the mapping is locked. + /// + void ClearStrings(); + + /// + /// Add strings from the given enumeration to the mapping. + /// + /// The strings to add. + /// The source provider of the strings to be logged. + void AddStrings(IEnumerable strings, string providerName); + + /// + /// Implements . + /// Specifies that this implementation handles strings. + /// + bool Handles(Type type); + + /// + /// Implements . + /// + IEnumerable GetSubtypes(Type type); + + /// + /// Implements . + /// + /// + MethodInfo GetStaticWriter(Type type); + + /// + /// Implements . + /// + /// + MethodInfo GetStaticReader(Type type); + + /// + /// Write the encoding of the given string to the stream. + /// + /// The stream to write to. + /// The (possibly null) string to write. + void WriteMappedString(Stream stream, string? value); + + /// + /// Try to read a string from the given stream. + /// + /// The stream to read from. + /// The (possibly null) string read. + /// + /// Thrown if the mapping is not locked. + /// + void ReadMappedString(Stream stream, out string? value); + + /// + /// See . + /// + event Action? ClientHandshakeComplete; + + } + +} diff --git a/Robust.Shared/Localization/LocalizationManager.cs b/Robust.Shared/Localization/LocalizationManager.cs index 57aec23f6..5a298edae 100644 --- a/Robust.Shared/Localization/LocalizationManager.cs +++ b/Robust.Shared/Localization/LocalizationManager.cs @@ -164,8 +164,8 @@ namespace Robust.Shared.Localization { _readEntry(entry, catalog); } - - RobustSerializer.MappedStringSerializer.AddStrings(yamlStream, filePath.ToString()); + IoCManager.Resolve() + .AddStrings(yamlStream, filePath.ToString()); } private static void _readEntry(YamlMappingNode entry, Catalog catalog) diff --git a/Robust.Shared/Prototypes/PrototypeManager.cs b/Robust.Shared/Prototypes/PrototypeManager.cs index d901924b5..e7e0a8a8a 100644 --- a/Robust.Shared/Prototypes/PrototypeManager.cs +++ b/Robust.Shared/Prototypes/PrototypeManager.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Runtime.Serialization; using JetBrains.Annotations; +using Robust.Shared.Asynchronous; using Robust.Shared.Interfaces.Reflection; using Robust.Shared.Interfaces.Resources; using Robust.Shared.IoC; @@ -70,6 +71,9 @@ namespace Robust.Shared.Prototypes /// Registers a specific prototype name to be ignored. /// void RegisterIgnore(string name); + + event Action? LoadedData; + } /// @@ -214,7 +218,7 @@ namespace Robust.Shared.Prototypes var result = ((YamlStream? yamlStream, ResourcePath?))(yamlStream, filePath); - RobustSerializer.MappedStringSerializer.AddStrings(yamlStream, filePath.ToString()); + LoadedData?.Invoke(yamlStream, filePath.ToString()); return result; } @@ -261,8 +265,7 @@ namespace Robust.Shared.Prototypes } } - - RobustSerializer.MappedStringSerializer.AddStrings(yaml, "anonymous prototypes YAML stream"); + LoadedData?.Invoke(yaml, "anonymous prototypes YAML stream"); } #endregion IPrototypeManager members @@ -355,6 +358,9 @@ namespace Robust.Shared.Prototypes { IgnoredPrototypeTypes.Add(name); } + + public event Action? LoadedData; + } [Serializable] diff --git a/Robust.Shared/Robust.Shared.csproj b/Robust.Shared/Robust.Shared.csproj index bf1507b72..588551034 100644 --- a/Robust.Shared/Robust.Shared.csproj +++ b/Robust.Shared/Robust.Shared.csproj @@ -35,18 +35,6 @@ - - RobustSerializer.MappedStringSerializer.cs - - - RobustSerializer.MappedStringSerializer.cs - - - RobustSerializer.MappedStringSerializer.cs - - - RobustSerializer.cs - RobustSerializer.cs diff --git a/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerClientHandshake.cs b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerClientHandshake.cs new file mode 100644 index 000000000..7a0c88fb8 --- /dev/null +++ b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerClientHandshake.cs @@ -0,0 +1,45 @@ +using JetBrains.Annotations; +using Lidgren.Network; +using Robust.Shared.Interfaces.Network; +using Robust.Shared.Network; + +namespace Robust.Shared.Serialization +{ + + /// + /// The client part of the string-exchange handshake, sent after the + /// client receives the mapping hash and after the client receives a + /// strings package. Tells the server if the client needs an updated + /// copy of the mapping. + /// + /// + /// Also sent by the client after a new copy of the string mapping + /// has been received. If successfully loaded, the value of + /// is false, otherwise it will be + /// true. + /// + /// + [UsedImplicitly] + internal class MsgRobustMappedStringsSerializerClientHandshake : NetMessage + { + + public MsgRobustMappedStringsSerializerClientHandshake(INetChannel ch) + : base(nameof(MsgRobustMappedStringsSerializerClientHandshake), MsgGroups.Core) + { + } + + /// + /// true if the client needs a new copy of the mapping, + /// false otherwise. + /// + public bool NeedsStrings { get; set; } + + public override void ReadFromBuffer(NetIncomingMessage buffer) + => NeedsStrings = buffer.ReadBoolean(); + + public override void WriteToBuffer(NetOutgoingMessage buffer) + => buffer.Write(NeedsStrings); + + } + +} diff --git a/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerServerHandshake.cs b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerServerHandshake.cs new file mode 100644 index 000000000..7ab23887b --- /dev/null +++ b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerServerHandshake.cs @@ -0,0 +1,54 @@ +using System; +using JetBrains.Annotations; +using Lidgren.Network; +using Robust.Shared.Interfaces.Network; +using Robust.Shared.Network; + +namespace Robust.Shared.Serialization +{ + /// + /// The server part of the string-exchange handshake. Sent as the + /// first message in the handshake. Tells the client the hash of + /// the current string mapping, so the client can check if it has + /// a local copy. + /// + /// + [UsedImplicitly] + internal class MsgRobustMappedStringsSerializerServerHandshake : NetMessage + { + + public MsgRobustMappedStringsSerializerServerHandshake(INetChannel ch) + : base(nameof(MsgRobustMappedStringsSerializerServerHandshake), MsgGroups.Core) + { + } + + /// + /// The hash of the current string mapping held by the server. + /// + public byte[]? Hash { get; set; } + + public override void ReadFromBuffer(NetIncomingMessage buffer) + { + var len = buffer.ReadVariableInt32(); + if (len > 64) + { + throw new InvalidOperationException("Hash too long."); + } + + Hash = buffer.ReadBytes(len); + } + + public override void WriteToBuffer(NetOutgoingMessage buffer) + { + if (Hash == null) + { + throw new InvalidOperationException("Package has not been specified."); + } + + buffer.WriteVariableInt32(Hash.Length); + buffer.Write(Hash); + } + + } + +} diff --git a/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerStrings.cs b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerStrings.cs new file mode 100644 index 000000000..c4eb187a4 --- /dev/null +++ b/Robust.Shared/Serialization/Messages/MsgRobustMappedStringsSerializerStrings.cs @@ -0,0 +1,61 @@ +using System; +using System.IO; +using JetBrains.Annotations; +using Lidgren.Network; +using Robust.Shared.Interfaces.Network; +using Robust.Shared.Network; + +namespace Robust.Shared.Serialization +{ + /// + /// The meat of the string-exchange handshake sandwich. Sent by the + /// server after the client requests an updated copy of the mapping. + /// Contains the updated string mapping. + /// + /// + [UsedImplicitly] + internal class MsgRobustMappedStringsSerializerStrings : NetMessage + { + + public MsgRobustMappedStringsSerializerStrings(INetChannel ch) + : base(nameof(MsgRobustMappedStringsSerializerStrings), MsgGroups.Core) + { + } + + /// + /// The raw bytes of the string mapping held by the server. + /// + public byte[]? Package { get; set; } + + public override void ReadFromBuffer(NetIncomingMessage buffer) + { + var l = buffer.ReadVariableInt32(); + var success = buffer.ReadBytes(l, out var buf); + if (!success) + { + throw new InvalidDataException("Not all of the bytes were available in the message."); + } + + Package = buf; + } + + public override void WriteToBuffer(NetOutgoingMessage buffer) + { + if (Package == null) + { + throw new InvalidOperationException("Package has not been specified."); + } + + buffer.WriteVariableInt32(Package.Length); + var start = buffer.LengthBytes; + buffer.Write(Package); + var added = buffer.LengthBytes - start; + if (added != Package.Length) + { + throw new InvalidOperationException("Not all of the bytes were written to the message."); + } + } + + } + +} diff --git a/Robust.Shared/Serialization/RobustMappedStringSerializer.cs b/Robust.Shared/Serialization/RobustMappedStringSerializer.cs new file mode 100644 index 000000000..a1c0bd457 --- /dev/null +++ b/Robust.Shared/Serialization/RobustMappedStringSerializer.cs @@ -0,0 +1,1319 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using JetBrains.Annotations; +using NetSerializer; +using Newtonsoft.Json.Linq; +using Robust.Shared.ContentPack; +using Robust.Shared.Interfaces.Log; +using Robust.Shared.Interfaces.Network; +using Robust.Shared.IoC; +using Robust.Shared.Log; +using Robust.Shared.Utility; +using YamlDotNet.RepresentationModel; + +namespace Robust.Shared.Serialization +{ + + /// + /// Serializer which manages a mapping of pre-loaded strings to constant + /// values, for message compression. The mapping is shared between the + /// server and client. + /// + /// + /// Strings are long and expensive to send over the wire, and lots of + /// strings involved in messages are sent repeatedly between the server + /// and client - such as filenames, icon states, constant strings, etc. + /// + /// To compress these strings, we use a constant string mapping, decided + /// by the server when it starts up, that associates strings with a + /// fixed value. The mapping is shared with clients when they connect. + /// + /// When sending these strings over the wire, the serializer can then + /// send the constant value instead - and at the other end, the + /// serializer can use the same mapping to recover the original string. + /// + public class RobustMappedStringSerializer : IStaticTypeSerializer, IRobustMappedStringSerializer + { + + private INetManager? _net; + + private readonly Lazy _lazyLogSzr = new Lazy(() => Logger.GetSawmill("szr")); + + private ISawmill LogSzr => _lazyLogSzr.Value; + + private readonly HashSet _incompleteHandshakes = new HashSet(); + + /// + /// Starts the handshake from the server end of the given channel, + /// sending a . + /// + /// The network channel to perform the handshake over. + /// + /// Locks the string mapping if this is the first time the server is + /// performing the handshake. + /// + /// + /// + public async Task Handshake(INetChannel channel) + { + var net = channel.NetPeer; + + if (net.IsClient) + { + return; + } + + if (!LockMappedStrings) + { + LockMappedStrings = true; + LogSzr.Info($"Locked in at {_mappedStrings.Count} mapped strings."); + } + + _incompleteHandshakes.Add(channel); + + var message = net.CreateNetMessage(); + message.Hash = MappedStringsHash; + net.ServerSendMessage(message, channel); + + while (_incompleteHandshakes.Contains(channel)) + { + await Task.Delay(1); + } + + LogSzr.Info($"Completed handshake with {channel.RemoteEndPoint.Address}."); + } + + /// + /// Performs the setup so that the serializer can perform the string- + /// exchange protocol. + /// + /// + /// The string-exchange protocol is started by the server when the + /// client first connects. The server sends the client a hash of the + /// string mapping; the client checks that hash against any local + /// caches; and if necessary, the client requests a new copy of the + /// mapping from the server. + /// + /// Uncached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// Cached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Dont Need Strings -> | + /// + /// + /// Verification failure flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// + Hash Failed | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// NOTE: Verification failure flow is currently not implemented. + /// + /// + /// The to perform the protocol steps over. + /// + /// + /// + /// + /// + /// + /// + /// + public void NetworkInitialize(INetManager net) + { + _net = net; + + net.RegisterNetMessage( + nameof(MsgRobustMappedStringsSerializerServerHandshake), + msg => HandleServerHandshake(net, msg)); + + net.RegisterNetMessage( + nameof(MsgRobustMappedStringsSerializerClientHandshake), + msg => HandleClientHandshake(net, msg)); + + net.RegisterNetMessage( + nameof(MsgRobustMappedStringsSerializerStrings), + msg => HandleStringsMessage(net, msg)); + } + + /// + /// Handles the reception, verification of a strings package + /// and subsequent mapping of strings and initiator of + /// receipt response. + /// + /// Uncached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | <- you are here on client + /// + /// Verification failure flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | + /// | <----------- Strings | + /// + Hash Failed | <- you are here on client + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | <- you are here on client + /// + /// + /// NOTE: Verification failure flow is currently not implemented. + /// + /// + /// Unable to verify strings package by hash. + /// + private void HandleStringsMessage(INetManager net, MsgRobustMappedStringsSerializerStrings msgRobustMappedStringsSerializer) + { + if (net.IsServer) + { + LogSzr.Error("Received strings from client."); + return; + } + + LockMappedStrings = false; + ClearStrings(); + DebugTools.Assert(msgRobustMappedStringsSerializer.Package != null, "msg.Package != null"); + LoadStrings(new MemoryStream(msgRobustMappedStringsSerializer.Package!, false)); + var checkHash = CalculateHash(msgRobustMappedStringsSerializer.Package!); + if (!checkHash.SequenceEqual(ServerHash)) + { + // TODO: retry sending MsgClientHandshake with NeedsStrings = false + throw new InvalidOperationException("Unable to verify strings package by hash." + $"\n{ConvertToBase64Url(checkHash)} vs. {ConvertToBase64Url(ServerHash)}"); + } + + _stringMapHash = ServerHash; + LockMappedStrings = true; + + LogSzr.Info($"Locked in at {_mappedStrings.Count} mapped strings."); + + WriteStringCache(); + + // ok we're good now + var channel = msgRobustMappedStringsSerializer.MsgChannel; + OnClientCompleteHandshake(net, channel); + } + + /// + /// Interpret a client's handshake, either sending a package + /// of strings or completing the handshake. + /// + /// Uncached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | <- you are here on server + /// | <----------- Strings | + /// | Dont Need Strings -> | <- you are here on server + /// + /// + /// Cached flow: + /// Client | Server + /// | <-------------- Hash | + /// | Dont Need Strings -> | <- you are here on server + /// + /// + /// Verification failure flow: + /// Client | Server + /// | <-------------- Hash | + /// | Need Strings ------> | <- you are here on server + /// | <----------- Strings | + /// + Hash Failed | + /// | Need Strings ------> | <- you are here on server + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// NOTE: Verification failure flow is currently not implemented. + /// + /// + private void HandleClientHandshake(INetManager net, MsgRobustMappedStringsSerializerClientHandshake msgRobustMappedStringsSerializer) + { + if (net.IsClient) + { + LogSzr.Error("Received client handshake on client."); + return; + } + + LogSzr.Info($"Received handshake from {msgRobustMappedStringsSerializer.MsgChannel.RemoteEndPoint.Address}."); + + if (!msgRobustMappedStringsSerializer.NeedsStrings) + { + LogSzr.Info($"Completing handshake with {msgRobustMappedStringsSerializer.MsgChannel.RemoteEndPoint.Address}."); + _incompleteHandshakes.Remove(msgRobustMappedStringsSerializer.MsgChannel); + return; + } + + // TODO: count and limit number of requests to send strings during handshake + + var strings = msgRobustMappedStringsSerializer.MsgChannel.NetPeer.CreateNetMessage(); + using (var ms = new MemoryStream()) + { + WriteStringPackage(ms); + ms.Position = 0; + strings.Package = ms.ToArray(); + LogSzr.Info($"Sending {ms.Length} bytes sized mapped strings package to {msgRobustMappedStringsSerializer.MsgChannel.RemoteEndPoint.Address}."); + } + + msgRobustMappedStringsSerializer.MsgChannel.SendMessage(strings); + } + + /// + /// Interpret a server's handshake, either requesting a package + /// of strings or completing the handshake. + /// + /// Uncached flow: + /// Client | Server + /// | <-------------- Hash | <- you are here on client + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// Cached flow: + /// Client | Server + /// | <-------------- Hash | <- you are here on client + /// | Dont Need Strings -> | + /// + /// + /// Verification failure flow: + /// Client | Server + /// | <-------------- Hash | <- you are here on client + /// | Need Strings ------> | + /// | <----------- Strings | + /// + Hash Failed | + /// | Need Strings ------> | + /// | <----------- Strings | + /// | Dont Need Strings -> | + /// + /// + /// NOTE: Verification failure flow is currently not implemented. + /// + /// Mapped strings are locked. + /// + private void HandleServerHandshake(INetManager net, MsgRobustMappedStringsSerializerServerHandshake msgRobustMappedStringsSerializer) + { + if (net.IsServer) + { + LogSzr.Error("Received server handshake on server."); + return; + } + + ServerHash = msgRobustMappedStringsSerializer.Hash; + LockMappedStrings = false; + + if (LockMappedStrings) + { + throw new InvalidOperationException("Mapped strings are locked."); + } + + ClearStrings(); + + var hashStr = ConvertToBase64Url(Convert.ToBase64String(msgRobustMappedStringsSerializer.Hash!)); + + LogSzr.Info($"Received server handshake with hash {hashStr}."); + + var fileName = CacheForHash(hashStr); + if (!File.Exists(fileName)) + { + LogSzr.Info($"No string cache for {hashStr}."); + var handshake = net.CreateNetMessage(); + LogSzr.Info("Asking server to send mapped strings."); + handshake.NeedsStrings = true; + msgRobustMappedStringsSerializer.MsgChannel.SendMessage(handshake); + } + else + { + LogSzr.Info($"We had a cached string map that matches {hashStr}."); + using var file = File.OpenRead(fileName); + var added = LoadStrings(file); + + _stringMapHash = msgRobustMappedStringsSerializer.Hash!; + LogSzr.Info($"Read {added} strings from cache {hashStr}."); + LockMappedStrings = true; + LogSzr.Info($"Locked in at {_mappedStrings.Count} mapped strings."); + // ok we're good now + var channel = msgRobustMappedStringsSerializer.MsgChannel; + OnClientCompleteHandshake(net, channel); + } + } + + /// + /// Inform the server that the client has a complete copy of the + /// mapping, and alert other code that the handshake is over. + /// + /// + /// + private void OnClientCompleteHandshake(INetManager net, INetChannel channel) + { + LogSzr.Info("Letting server know we're good to go."); + var handshake = net.CreateNetMessage(); + handshake.NeedsStrings = false; + channel.SendMessage(handshake); + + if (ClientHandshakeComplete == null) + { + LogSzr.Warning("There's no handler attached to ClientHandshakeComplete."); + } + + ClientHandshakeComplete?.Invoke(); + } + + /// + /// Gets the cache file associated with the given hash. + /// + /// The hash to look up the cache for. + /// + /// The filename where the cache for the given hash would be. The + /// file itself may or may not exist. If it does not exist, no cache + /// was made for the given hash. + /// + private string CacheForHash(string hashStr) + => PathHelpers.ExecutableRelativeFile($"strings-{hashStr}"); + + /// + /// Saves the string cache to a file based on it's hash. + /// + private void WriteStringCache() + { + var hashStr = Convert.ToBase64String(MappedStringsHash); + hashStr = ConvertToBase64Url(hashStr); + + var fileName = CacheForHash(hashStr); + using var file = File.OpenWrite(fileName); + WriteStringPackage(file); + + LogSzr.Info($"Wrote string cache {hashStr}."); + } + + private byte[]? _mappedStringsPackage; + + private byte[] MappedStringsPackage => LockMappedStrings + ? _mappedStringsPackage ??= WriteStringPackage() + : throw new InvalidOperationException("Mapped strings must be locked."); + + /// + /// Writes strings to a package and converts to an array of bytes. + /// + /// + /// This is invoked by accessing for the first time. + /// + private byte[] WriteStringPackage() + { + using var ms = new MemoryStream(); + WriteStringPackage(ms); + return ms.ToArray(); + } + + /// + /// Writes a strings package to a stream. + /// + /// A writable stream. + /// Overly long string in strings package. + public void WriteStringPackage(Stream stream) + { + // ReSharper disable once SuggestVarOrType_Elsewhere + Span buf = stackalloc byte[MaxMappedStringSize]; + var sw = Stopwatch.StartNew(); + var enc = Encoding.UTF8.GetEncoder(); + + using (var zs = new DeflateStream(stream, CompressionLevel.Optimal, true)) + { + var bytesWritten = WriteCompressedUnsignedInt(zs, (uint) MappedStrings.Count); + foreach (var str in MappedStrings) + { + if (str.Length >= MaxMappedStringSize) + { + throw new NotImplementedException("Overly long string in strings package."); + } + + var l = enc.GetBytes(str, buf, true); + + if (l >= MaxMappedStringSize) + { + throw new NotImplementedException("Overly long string in strings package."); + } + + bytesWritten += WriteCompressedUnsignedInt(zs, (uint) l); + + zs.Write(buf.Slice(0,l)); + + bytesWritten += l; + + enc.Reset(); + } + + zs.Write(BitConverter.GetBytes(bytesWritten)); + zs.Flush(); + } + + LogSzr.Info($"Wrote {MappedStrings.Count} strings to package in {sw.ElapsedMilliseconds}ms."); + } + + /// + /// Loads a strings package from a stream. + /// + /// + /// Uses to extract strings and adds them to the mapping. + /// + /// A readable stream. + /// The number of strings loaded. + /// Mapped strings are locked, will not load. + /// Did not read all bytes in package! + private int LoadStrings(Stream stream) + { + if (LockMappedStrings) + { + throw new InvalidOperationException("Mapped strings are locked, will not load."); + } + + var started = MappedStrings.Count; + foreach (var str in ReadStringPackage(stream)) + { + _stringMapping[str] = _mappedStrings.Count; + _mappedStrings.Add(str); + } + + if (stream.CanSeek && stream.CanRead) + { + if (stream.Position != stream.Length) + { + throw new InvalidDataException("Did not read all bytes in package!"); + } + } + + var added = MappedStrings.Count - started; + return added; + } + + /// + /// Reads the contents of a strings package. + /// + /// + /// Does not add strings to the current mapping. + /// + /// A readable stream. + /// Strings from within the package. + /// Could not read the full length of string #N. + private IEnumerable ReadStringPackage(Stream stream) + { + var buf = ArrayPool.Shared.Rent(65536); + var sw = Stopwatch.StartNew(); + using var zs = new DeflateStream(stream, CompressionMode.Decompress); + + var c = ReadCompressedUnsignedInt(zs, out var x); + var bytesRead = x; + for (var i = 0; i < c; ++i) + { + var l = (int) ReadCompressedUnsignedInt(zs, out x); + bytesRead += x; + var y = zs.Read(buf, 0, l); + if (y != l) + { + throw new InvalidDataException($"Could not read the full length of string #{i}."); + } + + bytesRead += y; + var str = Encoding.UTF8.GetString(buf, 0, l); + yield return str; + } + + zs.Read(buf, 0, 4); + var checkBytesRead = BitConverter.ToInt32(buf, 0); + if (checkBytesRead != bytesRead) + { + throw new InvalidDataException("Could not verify package was read correctly."); + } + + LogSzr.Info($"Read package of {c} strings in {sw.ElapsedMilliseconds}ms."); + } + + /// + /// Converts a byte array such as a hash to a Base64 representation that is URL safe. + /// + /// + /// A base64url string form of the byte array. + private string ConvertToBase64Url(byte[]? data) + => data == null ? "" : ConvertToBase64Url(Convert.ToBase64String(data)); + + /// + /// Converts a a Base64 string to one that is URL safe. + /// + /// A base64url formed string. + private string ConvertToBase64Url(string b64Str) + { + if (b64Str is null) + { + throw new ArgumentNullException(nameof(b64Str)); + } + + var cut = b64Str[^1] == '=' ? b64Str[^2] == '=' ? 2 : 1 : 0; + b64Str = new StringBuilder(b64Str).Replace('+', '-').Replace('/', '_').ToString(0, b64Str.Length - cut); + return b64Str; + } + + /// + /// Converts a URL-safe Base64 string into a byte array. + /// + /// A base64url formed string. + /// The represented byte array. + public byte[] ConvertFromBase64Url(string s) + { + var l = s.Length % 3; + var sb = new StringBuilder(s); + sb.Replace('-', '+').Replace('_', '/'); + for (var i = 0; i < l; ++i) + { + sb.Append('='); + } + + s = sb.ToString(); + return Convert.FromBase64String(s); + } + + public byte[]? ServerHash; + + private readonly IList _mappedStrings = new List(); + + private readonly IDictionary _stringMapping = new Dictionary(); + + public IReadOnlyList MappedStrings => new ReadOnlyCollection(_mappedStrings); + + /// + /// Whether the string mapping is decided, and cannot be changed. + /// + /// + /// + /// While false, strings can be added to the mapping, but + /// it cannot be saved to a cache. + /// + /// + /// While true, the mapping cannot be modified, but can be + /// shared between the server and client and saved to a cache. + /// + /// + public bool LockMappedStrings { get; set; } + + private readonly Regex _rxSymbolSplitter + = new Regex( + @"(?<=[^\s\W])(?=[A-Z]) # Match for split at start of new capital letter + |(?<=[^0-9\s\W])(?=[0-9]) # Match for split before spans of numbers + |(?<=[A-Za-z0-9])(?=_) # Match for a split before an underscore + |(?=[.\\\/,#$?!@|&*()^`""'`~[\]{}:;\-]) # Match for a split after symbols + |(?<=[.\\\/,#$?!@|&*()^`""'`~[\]{}:;\-]) # Match for a split before symbols too", + RegexOptions.CultureInvariant + | RegexOptions.Compiled + | RegexOptions.IgnorePatternWhitespace + ); + + /// + /// Add a string to the constant mapping. + /// + /// + /// If the string has multiple detectable subcomponents, such as a + /// filepath, it may result in more than one string being added to + /// the mapping. As string parts are commonly sent as subsets or + /// scoped names, this increases the likelyhood of a successful + /// string mapping. + /// + /// + /// true if the string was added to the mapping for the first + /// time, false otherwise. + /// + /// + /// Thrown if the string is not normalized (). + /// + public bool AddString(string str) + { + if (LockMappedStrings) + { + if (_net!.IsClient) + { + //LogSzr.Info("On client and mapped strings are locked, will not add."); + return false; + } + + //throw new InvalidOperationException("Mapped strings are locked, will not add."); + LogSzr.Info("On server and mapped strings are locked, will not add."); + return false; + } + + if (String.IsNullOrEmpty(str)) + { + return false; + } + + if (!str.IsNormalized()) + { + throw new InvalidOperationException("Only normalized strings may be added."); + } + + if (_stringMapping.ContainsKey(str)) + { + return false; + } + + if (str.Length >= MaxMappedStringSize) return false; + + if (str.Length <= MinMappedStringSize) return false; + + str = str.Trim(); + + if (str.Length <= MinMappedStringSize) return false; + + str = str.Replace(Environment.NewLine, "\n"); + + if (str.Length <= MinMappedStringSize) return false; + + var symTrimmedStr = str.Trim(TrimmableSymbolChars); + if (symTrimmedStr != str) + { + AddString(symTrimmedStr); + } + + if (str.Contains('/')) + { + var parts = str.Split('/', StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < parts.Length; ++i) + { + for (var l = 1; l <= parts.Length - i; ++l) + { + var subStr = String.Join('/', parts.Skip(i).Take(l)); + if (_stringMapping.TryAdd(subStr, _mappedStrings.Count)) + { + _mappedStrings.Add(subStr); + } + + if (!subStr.Contains('.')) + { + continue; + } + + var subParts = subStr.Split('.', StringSplitOptions.RemoveEmptyEntries); + for (var si = 0; si < subParts.Length; ++si) + { + for (var sl = 1; sl <= subParts.Length - si; ++sl) + { + var subSubStr = String.Join('.', subParts.Skip(si).Take(sl)); + // ReSharper disable once InvertIf + if (_stringMapping.TryAdd(subSubStr, _mappedStrings.Count)) + { + _mappedStrings.Add(subSubStr); + } + } + } + } + } + } + else if (str.Contains("_")) + { + foreach (var substr in str.Split("_")) + { + AddString(substr); + } + } + else if (str.Contains(" ")) + { + foreach (var substr in str.Split(" ")) + { + if (substr == str) continue; + + AddString(substr); + } + } + else + { + var parts = _rxSymbolSplitter.Split(str); + foreach (var substr in parts) + { + if (substr == str) continue; + + AddString(substr); + } + + for (var si = 0; si < parts.Length; ++si) + { + for (var sl = 1; sl <= parts.Length - si; ++sl) + { + var subSubStr = String.Concat(parts.Skip(si).Take(sl)); + if (_stringMapping.TryAdd(subSubStr, _mappedStrings.Count)) + { + _mappedStrings.Add(subSubStr); + } + } + } + } + + if (_stringMapping.TryAdd(str, _mappedStrings.Count)) + { + _mappedStrings.Add(str); + } + + _stringMapHash = null; + _mappedStringsPackage = null; + return true; + } + + /// + /// Add the constant strings from an to the + /// mapping. + /// + /// The assembly from which to collect constant strings. + [MethodImpl(MethodImplOptions.Synchronized)] + public unsafe void AddStrings(Assembly asm) + { + if (LockMappedStrings) + { + if (_net!.IsClient) + { + //LogSzr.Info("On client and mapped strings are locked, will not add."); + return; + } + + //throw new InvalidOperationException("Mapped strings are locked, will not add ."); + LogSzr.Info("On server and mapped strings are locked, will not add."); + return; + } + + var started = MappedStrings.Count; + var sw = Stopwatch.StartNew(); + if (asm.TryGetRawMetadata(out var blob, out var len)) + { + var reader = new MetadataReader(blob, len); + var usrStrHandle = default(UserStringHandle); + do + { + var userStr = reader.GetUserString(usrStrHandle); + if (userStr != "") + { + AddString(String.Intern(userStr.Normalize())); + } + + usrStrHandle = reader.GetNextHandle(usrStrHandle); + } while (usrStrHandle != default); + + var strHandle = default(StringHandle); + do + { + var str = reader.GetString(strHandle); + if (str != "") + { + AddString(String.Intern(str.Normalize())); + } + + strHandle = reader.GetNextHandle(strHandle); + } while (strHandle != default); + } + + var added = MappedStrings.Count - started; + LogSzr.Info($"Mapping {added} strings from {asm.GetName().Name} took {sw.ElapsedMilliseconds}ms."); + } + + /// + /// Add strings from the given to the mapping. + /// + /// + /// Strings are taken from YAML anchors, tags, and leaf nodes. + /// + /// The YAML to collect strings from. + /// The stream name. Only used for logging. + [MethodImpl(MethodImplOptions.Synchronized)] + public void AddStrings(YamlStream yaml, string name) + { + if (LockMappedStrings) + { + if (_net!.IsClient) + { + //LogSzr.Info("On client and mapped strings are locked, will not add."); + return; + } + + //throw new InvalidOperationException("Mapped strings are locked, will not add."); + LogSzr.Info("On server and mapped strings are locked, will not add."); + return; + } + + var started = MappedStrings.Count; + var sw = Stopwatch.StartNew(); + foreach (var doc in yaml) + { + foreach (var node in doc.AllNodes) + { + var a = node.Anchor; + if (!String.IsNullOrEmpty(a)) + { + AddString(a); + } + + var t = node.Tag; + if (!String.IsNullOrEmpty(t)) + { + AddString(t); + } + + switch (node) + { + case YamlScalarNode scalar: + { + var v = scalar.Value; + if (String.IsNullOrEmpty(v)) + { + continue; + } + + AddString(v); + break; + } + } + } + } + + var added = MappedStrings.Count - started; + LogSzr.Info($"Mapping {added} strings from {name} took {sw.ElapsedMilliseconds}ms."); + } + + /// + /// Add strings from the given to the mapping. + /// + /// + /// Strings are taken from JSON property names and string nodes. + /// + /// The JSON to collect strings from. + /// The stream name. Only used for logging. + public void AddStrings(JObject obj, string name) + { + if (LockMappedStrings) + { + if (_net!.IsClient) + { + //LogSzr.Info("On client and mapped strings are locked, will not add."); + return; + } + + //throw new InvalidOperationException("Mapped strings are locked, will not add."); + LogSzr.Info("On server and mapped strings are locked, will not add."); + return; + } + + var started = MappedStrings.Count; + var sw = Stopwatch.StartNew(); + foreach (var node in obj.DescendantsAndSelf()) + { + switch (node) + { + case JValue value: + { + if (value.Type != JTokenType.String) + { + continue; + } + + var v = value.Value?.ToString(); + if (String.IsNullOrEmpty(v)) + { + continue; + } + + AddString(v); + break; + } + case JProperty prop: + { + var propName = prop.Name; + if (String.IsNullOrEmpty(propName)) + { + continue; + } + + AddString(propName); + break; + } + } + } + + var added = MappedStrings.Count - started; + LogSzr.Info($"Mapping {added} strings from {name} took {sw.ElapsedMilliseconds}ms."); + } + + /// + /// Remove all strings from the mapping, completely resetting it. + /// + /// + /// Thrown if the mapping is locked. + /// + public void ClearStrings() + { + if (LockMappedStrings) + { + throw new InvalidOperationException("Mapped strings are locked, will not clear."); + } + + _mappedStrings.Clear(); + _stringMapping.Clear(); + _stringMapHash = null; + } + + /// + /// Add strings from the given enumeration to the mapping. + /// + /// The strings to add. + /// The source provider of the strings to be logged. + [MethodImpl(MethodImplOptions.Synchronized)] + public void AddStrings(IEnumerable strings, string providerName) + { + if (LockMappedStrings) + { + if (_net!.IsClient) + { + //LogSzr.Info("On client and mapped strings are locked, will not add."); + return; + } + + //throw new InvalidOperationException("Mapped strings are locked, will not add."); + LogSzr.Info("On server and mapped strings are locked, will not add."); + return; + } + + var started = MappedStrings.Count; + foreach (var str in strings) + { + AddString(str); + } + + var added = MappedStrings.Count - started; + LogSzr.Info($"Mapping {added} strings from {providerName}."); + } + + private byte[]? _stringMapHash; + + /// + /// The hash of the string mapping. + /// + /// + /// Thrown if the mapping is not locked. + /// + public byte[] MappedStringsHash => _stringMapHash ??= CalculateMappedStringsHash(); + + private byte[] CalculateMappedStringsHash() + { + if (!LockMappedStrings) + { + throw new InvalidOperationException("String table should be locked before attempting to retrieve hash."); + } + + var sw = Stopwatch.StartNew(); + + var hash = CalculateHash(MappedStringsPackage); + + LogSzr.Info($"Hashing {MappedStrings.Count} strings took {sw.ElapsedMilliseconds}ms."); + LogSzr.Info($"Size: {MappedStringsPackage.Length} bytes, Hash: {ConvertToBase64Url(hash)}"); + return hash; + } + + /// + /// Creates a SHA512 hash of the given array of bytes. + /// + /// An array of bytes to be hashed. + /// A 512-bit (64-byte) hash result as an array of bytes. + /// + private byte[] CalculateHash(byte[] data) + { + if (data is null) + { + throw new ArgumentNullException(nameof(data)); + } + + using var hasher = SHA512.Create(); + var hash = hasher.ComputeHash(data); + return hash; + } + + /// + /// Implements . + /// Specifies that this implementation handles strings. + /// + public bool Handles(Type type) => type == typeof(string); + + /// + /// Implements . + /// + public IEnumerable GetSubtypes(Type type) => Type.EmptyTypes; + + /// + /// Implements . + /// + /// + public MethodInfo GetStaticWriter(Type type) => WriteMappedStringMethodInfo; + + /// + /// Implements . + /// + /// + public MethodInfo GetStaticReader(Type type) => ReadMappedStringMethodInfo; + + private delegate void WriteStringDelegate(Stream stream, string? value); + + private delegate void ReadStringDelegate(Stream stream, out string? value); + + private static readonly MethodInfo WriteMappedStringMethodInfo + = ((WriteStringDelegate) StaticWriteMappedString).Method; + + private static readonly MethodInfo ReadMappedStringMethodInfo + = ((ReadStringDelegate) StaticReadMappedString).Method; + + private static readonly char[] TrimmableSymbolChars = + { + '.', '\\', '/', ',', '#', '$', '?', '!', '@', '|', '&', + '*', '(', ')', '^', '`', '"', '\'', '`', '~', '[', ']', + '{', '}', ':', ';', '-' + }; + + /// + /// The shortest a string can be in order to be inserted in the mapping. + /// + /// + /// Strings below a certain length aren't worth compressing. + /// + private const int MinMappedStringSize = 3; + + /// + /// The longest a string can be in order to be inserted in the mapping. + /// + private const int MaxMappedStringSize = 420; + + /// + /// The special value corresponding to a null string in the + /// encoding. + /// + private const int MappedNull = 0; + + /// + /// The special value corresponding to a string which was not mapped. + /// This is followed by the bytes of the unmapped string. + /// + private const int UnmappedString = 1; + + /// + /// The first non-special value, used for encoding mapped strings. + /// + /// + /// Since previous values are taken by and + /// , this value is used to encode + /// mapped strings at an offset - in the encoding, a value + /// >= FirstMappedIndexStart represents the string with + /// mapping of that value - FirstMappedIndexStart. + /// + private const int FirstMappedIndexStart = 2; + + + /// + /// Write the encoding of the given string to the stream. + /// Static form of for use by . + /// + /// The stream to write to. + /// The (possibly null) string to write. + public static void StaticWriteMappedString(Stream stream, string? value) + { + var mss = IoCManager.Resolve(); + mss.WriteMappedString(stream, value); + } + + /// + /// Write the encoding of the given string to the stream. + /// + /// The stream to write to. + /// The (possibly null) string to write. + public void WriteMappedString(Stream stream, string? value) + { + if (!LockMappedStrings) + { + LogSzr.Warning("Performing unlocked string mapping."); + } + + if (value == null) + { + WriteCompressedUnsignedInt(stream, MappedNull); + return; + } + + if (_stringMapping.TryGetValue(value, out var mapping)) + { +#if DEBUG + if (mapping >= _mappedStrings.Count || mapping < 0) + { + throw new InvalidOperationException("A string mapping outside of the mapped string table was encountered."); + } +#endif + WriteCompressedUnsignedInt(stream, (uint) mapping + FirstMappedIndexStart); + //Logger.DebugS("szr", $"Encoded mapped string: {value}"); + return; + } + + // indicate not mapped + WriteCompressedUnsignedInt(stream, UnmappedString); + var buf = Encoding.UTF8.GetBytes(value); + //Logger.DebugS("szr", $"Encoded unmapped string: {value}"); + WriteCompressedUnsignedInt(stream, (uint) buf.Length); + stream.Write(buf); + } + + /// + /// Try to read a string from the given stream. + /// Static form of for use by . + /// + /// The stream to read from. + /// The (possibly null) string read. + /// + /// Thrown if the mapping is not locked. + /// + public static void StaticReadMappedString(Stream stream, out string? value) + { + var mss = IoCManager.Resolve(); + mss.ReadMappedString(stream, out value); + } + + /// + /// Try to read a string from the given stream. + /// + /// The stream to read from. + /// The (possibly null) string read. + /// + /// Thrown if the mapping is not locked. + /// + public void ReadMappedString(Stream stream, out string? value) + { + if (!LockMappedStrings) + { + throw new InvalidOperationException("Not performing unlocked string mapping."); + } + + var mapIndex = ReadCompressedUnsignedInt(stream, out _); + if (mapIndex == MappedNull) + { + value = null; + return; + } + + if (mapIndex == UnmappedString) + { + // not mapped + var length = checked((int)ReadCompressedUnsignedInt(stream, out _)); + // ReSharper disable once SuggestVarOrType_Elsewhere + Span buf = stackalloc byte[length]; + stream.Read(buf); + value = Encoding.UTF8.GetString(buf); + //Logger.DebugS("szr", $"Decoded unmapped string: {value}"); + + return; + } + + value = _mappedStrings[(int) mapIndex - FirstMappedIndexStart]; + //Logger.DebugS("szr", $"Decoded mapped string: {value}"); + } + + // TODO: move the below methods to some stream helpers class + +#if ROBUST_SERIALIZER_DISABLE_COMPRESSED_UINTS + public static int WriteCompressedUnsignedInt(Stream stream, uint value) + { + WriteUnsignedInt(stream, value); + return 4; + } + + public static uint ReadCompressedUnsignedInt(Stream stream, out int byteCount) + { + byteCount = 4; + return ReadUnsignedInt(stream); + } +#else + public static int WriteCompressedUnsignedInt(Stream stream, uint value) + { + var length = 1; + while (value >= 0x80) + { + stream.WriteByte((byte) (0x80 | value)); + value >>= 7; + ++length; + } + + stream.WriteByte((byte) value); + return length; + } + + public static uint ReadCompressedUnsignedInt(Stream stream, out int byteCount) + { + byteCount = 0; + var value = 0u; + var shift = 0; + while (stream.CanRead) + { + var current = stream.ReadByte(); + ++byteCount; + if (current == -1) + { + throw new EndOfStreamException(); + } + + value |= (0x7Fu & (byte) current) << shift; + shift += 7; + if ((0x80 & current) == 0) + { + return value; + } + } + + throw new EndOfStreamException(); + } +#endif + + [UsedImplicitly] + public static unsafe void WriteUnsignedInt(Stream stream, uint value) + { + var bytes = MemoryMarshal.AsBytes(new ReadOnlySpan(&value, 1)); + stream.Write(bytes); + } + + [UsedImplicitly] + public static unsafe uint ReadUnsignedInt(Stream stream) + { + uint value; + var bytes = MemoryMarshal.AsBytes(new Span(&value, 1)); + stream.Read(bytes); + return value; + } + + /// + /// See . + /// + public event Action? ClientHandshakeComplete; + + } + +} diff --git a/Robust.Shared/Serialization/RobustSerializer.Handshake.cs b/Robust.Shared/Serialization/RobustSerializer.Handshake.cs index fdc678f8e..6e672a6c6 100644 --- a/Robust.Shared/Serialization/RobustSerializer.Handshake.cs +++ b/Robust.Shared/Serialization/RobustSerializer.Handshake.cs @@ -16,7 +16,7 @@ namespace Robust.Shared.Serialization /// /// public Task Handshake(INetChannel channel) - => MappedStringSerializer.Handshake(channel); + => _mappedStringSerializer.Handshake(channel); /// /// An event that occurs once all handshake extensions have @@ -26,8 +26,8 @@ namespace Robust.Shared.Serialization /// public event Action ClientHandshakeComplete { - add => MappedStringSerializer.ClientHandshakeComplete += value; - remove => MappedStringSerializer.ClientHandshakeComplete -= value; + add => _mappedStringSerializer.ClientHandshakeComplete += value; + remove => _mappedStringSerializer.ClientHandshakeComplete -= value; } } diff --git a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgClientHandshake.cs b/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgClientHandshake.cs deleted file mode 100644 index 374f0052d..000000000 --- a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgClientHandshake.cs +++ /dev/null @@ -1,54 +0,0 @@ -using JetBrains.Annotations; -using Lidgren.Network; -using Robust.Shared.Interfaces.Network; -using Robust.Shared.Network; - -namespace Robust.Shared.Serialization -{ - - public partial class RobustSerializer - { - - public partial class MappedStringSerializer - { - /// - /// The client part of the string-exchange handshake, sent after the - /// client receives the mapping hash and after the client receives a - /// strings package. Tells the server if the client needs an updated - /// copy of the mapping. - /// - /// - /// Also sent by the client after a new copy of the string mapping - /// has been received. If successfully loaded, the value of - /// is false, otherwise it will be - /// true. - /// - /// - [UsedImplicitly] - private class MsgClientHandshake : NetMessage - { - - public MsgClientHandshake(INetChannel ch) - : base($"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgClientHandshake)}", MsgGroups.Core) - { - } - - /// - /// true if the client needs a new copy of the mapping, - /// false otherwise. - /// - public bool NeedsStrings { get; set; } - - public override void ReadFromBuffer(NetIncomingMessage buffer) - => NeedsStrings = buffer.ReadBoolean(); - - public override void WriteToBuffer(NetOutgoingMessage buffer) - => buffer.Write(NeedsStrings); - - } - - } - - } - -} diff --git a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgServerHandshake.cs b/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgServerHandshake.cs deleted file mode 100644 index 132adebd5..000000000 --- a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgServerHandshake.cs +++ /dev/null @@ -1,65 +0,0 @@ -using System; -using JetBrains.Annotations; -using Lidgren.Network; -using Robust.Shared.Interfaces.Network; -using Robust.Shared.Network; - -namespace Robust.Shared.Serialization -{ - - public partial class RobustSerializer - { - - public partial class MappedStringSerializer - { - - /// - /// The server part of the string-exchange handshake. Sent as the - /// first message in the handshake. Tells the client the hash of - /// the current string mapping, so the client can check if it has - /// a local copy. - /// - /// - [UsedImplicitly] - private class MsgServerHandshake : NetMessage - { - - public MsgServerHandshake(INetChannel ch) - : base($"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgServerHandshake)}", MsgGroups.Core) - { - } - - /// - /// The hash of the current string mapping held by the server. - /// - public byte[]? Hash { get; set; } - - public override void ReadFromBuffer(NetIncomingMessage buffer) - { - var len = buffer.ReadVariableInt32(); - if (len > 64) - { - throw new InvalidOperationException("Hash too long."); - } - - Hash = buffer.ReadBytes(len); - } - - public override void WriteToBuffer(NetOutgoingMessage buffer) - { - if (Hash == null) - { - throw new InvalidOperationException("Package has not been specified."); - } - - buffer.WriteVariableInt32(Hash.Length); - buffer.Write(Hash); - } - - } - - } - - } - -} diff --git a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgStrings.cs b/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgStrings.cs deleted file mode 100644 index e55ef691e..000000000 --- a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.MsgStrings.cs +++ /dev/null @@ -1,72 +0,0 @@ -using System; -using System.IO; -using JetBrains.Annotations; -using Lidgren.Network; -using Robust.Shared.Interfaces.Network; -using Robust.Shared.Network; - -namespace Robust.Shared.Serialization -{ - - public partial class RobustSerializer - { - - public partial class MappedStringSerializer - { - - /// - /// The meat of the string-exchange handshake sandwich. Sent by the - /// server after the client requests an updated copy of the mapping. - /// Contains the updated string mapping. - /// - /// - [UsedImplicitly] - private class MsgStrings : NetMessage - { - - public MsgStrings(INetChannel ch) - : base($"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgStrings)}", MsgGroups.Core) - { - } - - /// - /// The raw bytes of the string mapping held by the server. - /// - public byte[]? Package { get; set; } - - public override void ReadFromBuffer(NetIncomingMessage buffer) - { - var l = buffer.ReadVariableInt32(); - var success = buffer.ReadBytes(l, out var buf); - if (!success) - { - throw new InvalidDataException("Not all of the bytes were available in the message."); - } - - Package = buf; - } - - public override void WriteToBuffer(NetOutgoingMessage buffer) - { - if (Package == null) - { - throw new InvalidOperationException("Package has not been specified."); - } - - buffer.WriteVariableInt32(Package.Length); - var start = buffer.LengthBytes; - buffer.Write(Package); - var added = buffer.LengthBytes - start; - if (added != Package.Length) - { - throw new InvalidOperationException("Not all of the bytes were written to the message."); - } - } - - } - - } - - } - -} diff --git a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.cs b/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.cs deleted file mode 100644 index 118507e97..000000000 --- a/Robust.Shared/Serialization/RobustSerializer.MappedStringSerializer.cs +++ /dev/null @@ -1,1295 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.Metadata.Ecma335; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using JetBrains.Annotations; -using NetSerializer; -using Newtonsoft.Json.Linq; -using Robust.Shared.ContentPack; -using Robust.Shared.Interfaces.Log; -using Robust.Shared.Interfaces.Network; -using Robust.Shared.IoC; -using Robust.Shared.Log; -using Robust.Shared.Utility; -using YamlDotNet.RepresentationModel; - -namespace Robust.Shared.Serialization -{ - - public partial class RobustSerializer - { - - /// - /// Serializer which manages a mapping of pre-loaded strings to constant - /// values, for message compression. The mapping is shared between the - /// server and client. - /// - /// - /// Strings are long and expensive to send over the wire, and lots of - /// strings involved in messages are sent repeatedly between the server - /// and client - such as filenames, icon states, constant strings, etc. - /// - /// To compress these strings, we use a constant string mapping, decided - /// by the server when it starts up, that associates strings with a - /// fixed value. The mapping is shared with clients when they connect. - /// - /// When sending these strings over the wire, the serializer can then - /// send the constant value instead - and at the other end, the - /// serializer can use the same mapping to recover the original string. - /// - public partial class MappedStringSerializer : IStaticTypeSerializer - { - - private static INetManager? _net; - - private static readonly ISawmill LogSzr = Logger.GetSawmill("szr"); - - private static readonly HashSet IncompleteHandshakes = new HashSet(); - - /// - /// Starts the handshake from the server end of the given channel, - /// sending a . - /// - /// The network channel to perform the handshake over. - /// - /// Locks the string mapping if this is the first time the server is - /// performing the handshake. - /// - /// - /// - public static async Task Handshake(INetChannel channel) - { - var net = channel.NetPeer; - - if (net.IsClient) - { - return; - } - - if (!LockMappedStrings) - { - LockMappedStrings = true; - LogSzr.Info($"Locked in at {_MappedStrings.Count} mapped strings."); - } - - IncompleteHandshakes.Add(channel); - - var message = net.CreateNetMessage(); - message.Hash = MappedStringsHash; - net.ServerSendMessage(message, channel); - - while (IncompleteHandshakes.Contains(channel)) - { - await Task.Delay(1); - } - - LogSzr.Info($"Completed handshake with {channel.RemoteEndPoint.Address}."); - } - - /// - /// Performs the setup so that the serializer can perform the string- - /// exchange protocol. - /// - /// - /// The string-exchange protocol is started by the server when the - /// client first connects. The server sends the client a hash of the - /// string mapping; the client checks that hash against any local - /// caches; and if necessary, the client requests a new copy of the - /// mapping from the server. - /// - /// Uncached flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | - /// - /// - /// Cached flow: - /// Client | Server - /// | <-------------- Hash | - /// | Dont Need Strings -> | - /// - /// - /// Verification failure flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | - /// | <----------- Strings | - /// + Hash Failed | - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | - /// - /// - /// NOTE: Verification failure flow is currently not implemented. - /// - /// - /// The to perform the protocol steps over. - /// - /// - /// - /// - /// - /// - /// - /// - public static void NetworkInitialize(INetManager net) - { - _net = net; - - net.RegisterNetMessage( - $"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgServerHandshake)}", - msg => HandleServerHandshake(net, msg)); - - net.RegisterNetMessage( - $"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgClientHandshake)}", - msg => HandleClientHandshake(net, msg)); - - net.RegisterNetMessage( - $"{nameof(RobustSerializer)}.{nameof(MappedStringSerializer)}.{nameof(MsgStrings)}", - msg => HandleStringsMessage(net, msg)); - } - - /// - /// Handles the reception, verification of a strings package - /// and subsequent mapping of strings and initiator of - /// receipt response. - /// - /// Uncached flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | <- you are here on client - /// - /// Verification failure flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | - /// | <----------- Strings | - /// + Hash Failed | <- you are here on client - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | <- you are here on client - /// - /// - /// NOTE: Verification failure flow is currently not implemented. - /// - /// - /// Unable to verify strings package by hash. - /// - private static void HandleStringsMessage(INetManager net, MsgStrings msg) - { - if (net.IsServer) - { - LogSzr.Error("Received strings from client."); - return; - } - - LockMappedStrings = false; - ClearStrings(); - DebugTools.Assert(msg.Package != null, "msg.Package != null"); - LoadStrings(new MemoryStream(msg.Package!, false)); - var checkHash = CalculateHash(msg.Package!); - if (!checkHash.SequenceEqual(ServerHash)) - { - // TODO: retry sending MsgClientHandshake with NeedsStrings = false - throw new InvalidOperationException("Unable to verify strings package by hash." + $"\n{ConvertToBase64Url(checkHash)} vs. {ConvertToBase64Url(ServerHash)}"); - } - - _stringMapHash = ServerHash; - LockMappedStrings = true; - - LogSzr.Info($"Locked in at {_MappedStrings.Count} mapped strings."); - - WriteStringCache(); - - // ok we're good now - var channel = msg.MsgChannel; - OnClientCompleteHandshake(net, channel); - } - - /// - /// Interpret a client's handshake, either sending a package - /// of strings or completing the handshake. - /// - /// Uncached flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | <- you are here on server - /// | <----------- Strings | - /// | Dont Need Strings -> | <- you are here on server - /// - /// - /// Cached flow: - /// Client | Server - /// | <-------------- Hash | - /// | Dont Need Strings -> | <- you are here on server - /// - /// - /// Verification failure flow: - /// Client | Server - /// | <-------------- Hash | - /// | Need Strings ------> | <- you are here on server - /// | <----------- Strings | - /// + Hash Failed | - /// | Need Strings ------> | <- you are here on server - /// | <----------- Strings | - /// | Dont Need Strings -> | - /// - /// - /// NOTE: Verification failure flow is currently not implemented. - /// - /// - private static void HandleClientHandshake(INetManager net, MsgClientHandshake msg) - { - if (net.IsClient) - { - LogSzr.Error("Received client handshake on client."); - return; - } - - LogSzr.Info($"Received handshake from {msg.MsgChannel.RemoteEndPoint.Address}."); - - if (!msg.NeedsStrings) - { - LogSzr.Info($"Completing handshake with {msg.MsgChannel.RemoteEndPoint.Address}."); - IncompleteHandshakes.Remove(msg.MsgChannel); - return; - } - - // TODO: count and limit number of requests to send strings during handshake - - var strings = msg.MsgChannel.NetPeer.CreateNetMessage(); - using (var ms = new MemoryStream()) - { - WriteStringPackage(ms); - ms.Position = 0; - strings.Package = ms.ToArray(); - LogSzr.Info($"Sending {ms.Length} bytes sized mapped strings package to {msg.MsgChannel.RemoteEndPoint.Address}."); - } - - msg.MsgChannel.SendMessage(strings); - } - - /// - /// Interpret a server's handshake, either requesting a package - /// of strings or completing the handshake. - /// - /// Uncached flow: - /// Client | Server - /// | <-------------- Hash | <- you are here on client - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | - /// - /// - /// Cached flow: - /// Client | Server - /// | <-------------- Hash | <- you are here on client - /// | Dont Need Strings -> | - /// - /// - /// Verification failure flow: - /// Client | Server - /// | <-------------- Hash | <- you are here on client - /// | Need Strings ------> | - /// | <----------- Strings | - /// + Hash Failed | - /// | Need Strings ------> | - /// | <----------- Strings | - /// | Dont Need Strings -> | - /// - /// - /// NOTE: Verification failure flow is currently not implemented. - /// - /// Mapped strings are locked. - /// - private static void HandleServerHandshake(INetManager net, MsgServerHandshake msg) - { - if (net.IsServer) - { - LogSzr.Error("Received server handshake on server."); - return; - } - - ServerHash = msg.Hash; - LockMappedStrings = false; - - if (LockMappedStrings) - { - throw new InvalidOperationException("Mapped strings are locked."); - } - - ClearStrings(); - - var hashStr = ConvertToBase64Url(Convert.ToBase64String(msg.Hash!)); - - LogSzr.Info($"Received server handshake with hash {hashStr}."); - - var fileName = CacheForHash(hashStr); - if (!File.Exists(fileName)) - { - LogSzr.Info($"No string cache for {hashStr}."); - var handshake = net.CreateNetMessage(); - LogSzr.Info("Asking server to send mapped strings."); - handshake.NeedsStrings = true; - msg.MsgChannel.SendMessage(handshake); - } - else - { - LogSzr.Info($"We had a cached string map that matches {hashStr}."); - using var file = File.OpenRead(fileName); - var added = LoadStrings(file); - - _stringMapHash = msg.Hash!; - LogSzr.Info($"Read {added} strings from cache {hashStr}."); - LockMappedStrings = true; - LogSzr.Info($"Locked in at {_MappedStrings.Count} mapped strings."); - // ok we're good now - var channel = msg.MsgChannel; - OnClientCompleteHandshake(net, channel); - } - } - - /// - /// Inform the server that the client has a complete copy of the - /// mapping, and alert other code that the handshake is over. - /// - /// - /// - private static void OnClientCompleteHandshake(INetManager net, INetChannel channel) - { - LogSzr.Info("Letting server know we're good to go."); - var handshake = net.CreateNetMessage(); - handshake.NeedsStrings = false; - channel.SendMessage(handshake); - - if (ClientHandshakeComplete == null) - { - LogSzr.Warning("There's no handler attached to ClientHandshakeComplete."); - } - - ClientHandshakeComplete?.Invoke(); - } - - /// - /// Gets the cache file associated with the given hash. - /// - /// The hash to look up the cache for. - /// - /// The filename where the cache for the given hash would be. The - /// file itself may or may not exist. If it does not exist, no cache - /// was made for the given hash. - /// - private static string CacheForHash(string hashStr) - => PathHelpers.ExecutableRelativeFile($"strings-{hashStr}"); - - /// - /// Saves the string cache to a file based on it's hash. - /// - private static void WriteStringCache() - { - var hashStr = Convert.ToBase64String(MappedStringsHash); - hashStr = ConvertToBase64Url(hashStr); - - var fileName = CacheForHash(hashStr); - using var file = File.OpenWrite(fileName); - WriteStringPackage(file); - - LogSzr.Info($"Wrote string cache {hashStr}."); - } - - private static byte[]? _mappedStringsPackage; - - private static byte[] MappedStringsPackage => LockMappedStrings - ? _mappedStringsPackage ??= WriteStringPackage() - : throw new InvalidOperationException("Mapped strings must be locked."); - - /// - /// Writes strings to a package and converts to an array of bytes. - /// - /// - /// This is invoked by accessing for the first time. - /// - private static byte[] WriteStringPackage() - { - using var ms = new MemoryStream(); - WriteStringPackage(ms); - return ms.ToArray(); - } - - /// - /// Strings longer than this will throw an exception and a better strategy will need to be employed to deal with large strings. - /// - public static int StringPackageMaximumBufferSize = 65536; - - /// - /// Writes a strings package to a stream. - /// - /// A writable stream. - /// Overly long string in strings package. - public static void WriteStringPackage(Stream stream) - { - var buf = new byte[StringPackageMaximumBufferSize]; - var sw = Stopwatch.StartNew(); - var enc = Encoding.UTF8.GetEncoder(); - - using (var zs = new DeflateStream(stream, CompressionLevel.Optimal, true)) - { - var bytesWritten = WriteCompressedUnsignedInt(zs, (uint) MappedStrings.Count); - foreach (var str in MappedStrings) - { - if (str.Length >= StringPackageMaximumBufferSize) - { - throw new NotImplementedException("Overly long string in strings package."); - } - - var l = enc.GetBytes(str, buf, true); - - if (l >= StringPackageMaximumBufferSize) - { - throw new NotImplementedException("Overly long string in strings package."); - } - - bytesWritten += WriteCompressedUnsignedInt(zs, (uint) l); - - zs.Write(buf, 0, l); - - bytesWritten += l; - - enc.Reset(); - } - - zs.Write(BitConverter.GetBytes(bytesWritten)); - zs.Flush(); - } - - LogSzr.Info($"Wrote {MappedStrings.Count} strings to package in {sw.ElapsedMilliseconds}ms."); - } - - /// - /// Loads a strings package from a stream. - /// - /// - /// Uses to extract strings and adds them to the mapping. - /// - /// A readable stream. - /// The number of strings loaded. - /// Mapped strings are locked, will not load. - /// Did not read all bytes in package! - private static int LoadStrings(Stream stream) - { - if (LockMappedStrings) - { - throw new InvalidOperationException("Mapped strings are locked, will not load."); - } - - var started = MappedStrings.Count; - foreach (var str in ReadStringPackage(stream)) - { - _StringMapping[str] = _MappedStrings.Count; - _MappedStrings.Add(str); - } - - if (stream.CanSeek && stream.CanRead) - { - if (stream.Position != stream.Length) - { - throw new InvalidDataException("Did not read all bytes in package!"); - } - } - - var added = MappedStrings.Count - started; - return added; - } - - /// - /// Reads the contents of a strings package. - /// - /// - /// Does not add strings to the current mapping. - /// - /// A readable stream. - /// Strings from within the package. - /// Could not read the full length of string #N. - private static IEnumerable ReadStringPackage(Stream stream) - { - var buf = ArrayPool.Shared.Rent(65536); - var sw = Stopwatch.StartNew(); - using var zs = new DeflateStream(stream, CompressionMode.Decompress); - - var c = ReadCompressedUnsignedInt(zs, out var x); - var bytesRead = x; - for (var i = 0; i < c; ++i) - { - var l = (int) ReadCompressedUnsignedInt(zs, out x); - bytesRead += x; - var y = zs.Read(buf, 0, l); - if (y != l) - { - throw new InvalidDataException($"Could not read the full length of string #{i}."); - } - - bytesRead += y; - var str = Encoding.UTF8.GetString(buf, 0, l); - yield return str; - } - - zs.Read(buf, 0, 4); - var checkBytesRead = BitConverter.ToInt32(buf, 0); - if (checkBytesRead != bytesRead) - { - throw new InvalidDataException("Could not verify package was read correctly."); - } - - LogSzr.Info($"Read package of {c} strings in {sw.ElapsedMilliseconds}ms."); - } - - /// - /// Converts a byte array such as a hash to a Base64 representation that is URL safe. - /// - /// - /// A base64url string form of the byte array. - private static string ConvertToBase64Url(byte[]? data) - => data == null ? "" : ConvertToBase64Url(Convert.ToBase64String(data)); - - /// - /// Converts a a Base64 string to one that is URL safe. - /// - /// A base64url formed string. - private static string ConvertToBase64Url(string b64Str) - { - if (b64Str is null) - { - throw new ArgumentNullException(nameof(b64Str)); - } - - var cut = b64Str[^1] == '=' ? b64Str[^2] == '=' ? 2 : 1 : 0; - b64Str = new StringBuilder(b64Str).Replace('+', '-').Replace('/', '_').ToString(0, b64Str.Length - cut); - return b64Str; - } - - /// - /// Converts a URL-safe Base64 string into a byte array. - /// - /// A base64url formed string. - /// The represented byte array. - public static byte[] ConvertFromBase64Url(string s) - { - var l = s.Length % 3; - var sb = new StringBuilder(s); - sb.Replace('-', '+').Replace('_', '/'); - for (var i = 0; i < l; ++i) - { - sb.Append('='); - } - - s = sb.ToString(); - return Convert.FromBase64String(s); - } - - public static byte[]? ServerHash; - - private static readonly IList _MappedStrings = new List(); - - private static readonly IDictionary _StringMapping = new Dictionary(); - - public static IReadOnlyList MappedStrings => new ReadOnlyCollection(_MappedStrings); - - /// - /// Whether the string mapping is decided, and cannot be changed. - /// - /// - /// - /// While false, strings can be added to the mapping, but - /// it cannot be saved to a cache. - /// - /// - /// While true, the mapping cannot be modified, but can be - /// shared between the server and client and saved to a cache. - /// - /// - public static bool LockMappedStrings { get; set; } - - private static readonly Regex RxSymbolSplitter - = new Regex( - @"(?<=[^\s\W])(?=[A-Z]) # Match for split at start of new capital letter - |(?<=[^0-9\s\W])(?=[0-9]) # Match for split before spans of numbers - |(?<=[A-Za-z0-9])(?=_) # Match for a split before an underscore - |(?=[.\\\/,#$?!@|&*()^`""'`~[\]{}:;\-]) # Match for a split after symbols - |(?<=[.\\\/,#$?!@|&*()^`""'`~[\]{}:;\-]) # Match for a split before symbols too", - RegexOptions.CultureInvariant - | RegexOptions.Compiled - | RegexOptions.IgnorePatternWhitespace - ); - - /// - /// Add a string to the constant mapping. - /// - /// - /// If the string has multiple detectable subcomponents, such as a - /// filepath, it may result in more than one string being added to - /// the mapping. As string parts are commonly sent as subsets or - /// scoped names, this increases the likelyhood of a successful - /// string mapping. - /// - /// - /// true if the string was added to the mapping for the first - /// time, false otherwise. - /// - /// - /// Thrown if the mapping is locked, and strings cannot be added, or - /// if the string is not normalized (). - /// - public static bool AddString(string str) - { - if (LockMappedStrings) - { - if (_net!.IsClient) - { - //LogSzr.Info("On client and mapped strings are locked, will not add."); - return false; - } - - throw new InvalidOperationException("Mapped strings are locked, will not add."); - } - - if (string.IsNullOrEmpty(str)) - { - return false; - } - - if (!str.IsNormalized()) - { - throw new InvalidOperationException("Only normalized strings may be added."); - } - - if (_StringMapping.ContainsKey(str)) - { - return false; - } - - if (str.Length >= MaxMappedStringSize) return false; - - if (str.Length <= MinMappedStringSize) return false; - - str = str.Trim(); - - if (str.Length <= MinMappedStringSize) return false; - - str = str.Replace(Environment.NewLine, "\n"); - - if (str.Length <= MinMappedStringSize) return false; - - var symTrimmedStr = str.Trim(TrimmableSymbolChars); - if (symTrimmedStr != str) - { - AddString(symTrimmedStr); - } - - if (str.Contains('/')) - { - var parts = str.Split('/', StringSplitOptions.RemoveEmptyEntries); - for (var i = 0; i < parts.Length; ++i) - { - for (var l = 1; l <= parts.Length - i; ++l) - { - var subStr = string.Join('/', parts.Skip(i).Take(l)); - if (_StringMapping.TryAdd(subStr, _MappedStrings.Count)) - { - _MappedStrings.Add(subStr); - } - - if (!subStr.Contains('.')) - { - continue; - } - - var subParts = subStr.Split('.', StringSplitOptions.RemoveEmptyEntries); - for (var si = 0; si < subParts.Length; ++si) - { - for (var sl = 1; sl <= subParts.Length - si; ++sl) - { - var subSubStr = string.Join('.', subParts.Skip(si).Take(sl)); - // ReSharper disable once InvertIf - if (_StringMapping.TryAdd(subSubStr, _MappedStrings.Count)) - { - _MappedStrings.Add(subSubStr); - } - } - } - } - } - } - else if (str.Contains("_")) - { - foreach (var substr in str.Split("_")) - { - AddString(substr); - } - } - else if (str.Contains(" ")) - { - foreach (var substr in str.Split(" ")) - { - if (substr == str) continue; - - AddString(substr); - } - } - else - { - var parts = RxSymbolSplitter.Split(str); - foreach (var substr in parts) - { - if (substr == str) continue; - - AddString(substr); - } - - for (var si = 0; si < parts.Length; ++si) - { - for (var sl = 1; sl <= parts.Length - si; ++sl) - { - var subSubStr = string.Concat(parts.Skip(si).Take(sl)); - if (_StringMapping.TryAdd(subSubStr, _MappedStrings.Count)) - { - _MappedStrings.Add(subSubStr); - } - } - } - } - - if (_StringMapping.TryAdd(str, _MappedStrings.Count)) - { - _MappedStrings.Add(str); - } - - _stringMapHash = null; - _mappedStringsPackage = null; - return true; - } - - /// - /// Add the constant strings from an to the - /// mapping. - /// - /// The assembly from which to collect constant strings. - /// - /// Thrown if the mapping is locked. - /// - [MethodImpl(MethodImplOptions.Synchronized)] - public static unsafe void AddStrings(Assembly asm) - { - if (LockMappedStrings) - { - if (_net!.IsClient) - { - //LogSzr.Info("On client and mapped strings are locked, will not add."); - return; - } - - throw new InvalidOperationException("Mapped strings are locked, will not add ."); - } - - var started = MappedStrings.Count; - var sw = Stopwatch.StartNew(); - if (asm.TryGetRawMetadata(out var blob, out var len)) - { - var reader = new MetadataReader(blob, len); - var usrStrHandle = default(UserStringHandle); - do - { - var userStr = reader.GetUserString(usrStrHandle); - if (userStr != "") - { - AddString(string.Intern(userStr.Normalize())); - } - - usrStrHandle = reader.GetNextHandle(usrStrHandle); - } while (usrStrHandle != default); - - var strHandle = default(StringHandle); - do - { - var str = reader.GetString(strHandle); - if (str != "") - { - AddString(string.Intern(str.Normalize())); - } - - strHandle = reader.GetNextHandle(strHandle); - } while (strHandle != default); - } - - var added = MappedStrings.Count - started; - LogSzr.Info($"Mapping {added} strings from {asm.GetName().Name} took {sw.ElapsedMilliseconds}ms."); - } - - /// - /// Add strings from the given to the mapping. - /// - /// - /// Strings are taken from YAML anchors, tags, and leaf nodes. - /// - /// The YAML to collect strings from. - /// The stream name. Only used for logging. - /// - /// Thrown if the mapping is locked. - /// - [MethodImpl(MethodImplOptions.Synchronized)] - public static void AddStrings(YamlStream yaml, string name) - { - if (LockMappedStrings) - { - if (_net!.IsClient) - { - //LogSzr.Info("On client and mapped strings are locked, will not add."); - return; - } - - throw new InvalidOperationException("Mapped strings are locked, will not add."); - } - - var started = MappedStrings.Count; - var sw = Stopwatch.StartNew(); - foreach (var doc in yaml) - { - foreach (var node in doc.AllNodes) - { - var a = node.Anchor; - if (!string.IsNullOrEmpty(a)) - { - AddString(a); - } - - var t = node.Tag; - if (!string.IsNullOrEmpty(t)) - { - AddString(t); - } - - switch (node) - { - case YamlScalarNode scalar: - { - var v = scalar.Value; - if (string.IsNullOrEmpty(v)) - { - continue; - } - - AddString(v); - break; - } - } - } - } - - var added = MappedStrings.Count - started; - LogSzr.Info($"Mapping {added} strings from {name} took {sw.ElapsedMilliseconds}ms."); - } - - /// - /// Add strings from the given to the mapping. - /// - /// - /// Strings are taken from JSON property names and string nodes. - /// - /// The JSON to collect strings from. - /// The stream name. Only used for logging. - /// - /// Thrown if the mapping is locked. - /// - public static void AddStrings(JObject obj, string name) - { - if (LockMappedStrings) - { - if (_net!.IsClient) - { - //LogSzr.Info("On client and mapped strings are locked, will not add."); - return; - } - - throw new InvalidOperationException("Mapped strings are locked, will not add."); - } - - var started = MappedStrings.Count; - var sw = Stopwatch.StartNew(); - foreach (var node in obj.DescendantsAndSelf()) - { - switch (node) - { - case JValue value: - { - if (value.Type != JTokenType.String) - { - continue; - } - - var v = value.Value?.ToString(); - if (string.IsNullOrEmpty(v)) - { - continue; - } - - AddString(v); - break; - } - case JProperty prop: - { - var propName = prop.Name; - if (string.IsNullOrEmpty(propName)) - { - continue; - } - - AddString(propName); - break; - } - } - } - - var added = MappedStrings.Count - started; - LogSzr.Info($"Mapping {added} strings from {name} took {sw.ElapsedMilliseconds}ms."); - } - - /// - /// Remove all strings from the mapping, completely resetting it. - /// - /// - /// Thrown if the mapping is locked. - /// - public static void ClearStrings() - { - if (LockMappedStrings) - { - throw new InvalidOperationException("Mapped strings are locked, will not clear."); - } - - _MappedStrings.Clear(); - _StringMapping.Clear(); - _stringMapHash = null; - } - - /// - /// Add strings from the given enumeration to the mapping. - /// - /// The strings to add. - /// The source provider of the strings to be logged. - /// - /// Thrown if the mapping is locked. - /// - [MethodImpl(MethodImplOptions.Synchronized)] - public static void AddStrings(IEnumerable strings, string providerName) - { - if (LockMappedStrings) - { - if (_net!.IsClient) - { - //LogSzr.Info("On client and mapped strings are locked, will not add."); - return; - } - - throw new InvalidOperationException("Mapped strings are locked, will not add."); - } - - var started = MappedStrings.Count; - foreach (var str in strings) - { - AddString(str); - } - - var added = MappedStrings.Count - started; - LogSzr.Info($"Mapping {added} strings from {providerName}."); - } - - private static byte[]? _stringMapHash; - - /// - /// The hash of the string mapping. - /// - /// - /// Thrown if the mapping is not locked. - /// - public static byte[] MappedStringsHash => _stringMapHash ??= CalculateMappedStringsHash(); - - private static byte[] CalculateMappedStringsHash() - { - if (!LockMappedStrings) - { - throw new InvalidOperationException("String table should be locked before attempting to retrieve hash."); - } - - var sw = Stopwatch.StartNew(); - - var hash = CalculateHash(MappedStringsPackage); - - LogSzr.Info($"Hashing {MappedStrings.Count} strings took {sw.ElapsedMilliseconds}ms."); - LogSzr.Info($"Size: {MappedStringsPackage.Length} bytes, Hash: {ConvertToBase64Url(hash)}"); - return hash; - } - - /// - /// Creates a SHA512 hash of the given array of bytes. - /// - /// An array of bytes to be hashed. - /// A 512-bit (64-byte) hash result as an array of bytes. - /// - private static byte[] CalculateHash(byte[] data) - { - if (data is null) - { - throw new ArgumentNullException(nameof(data)); - } - - using var hasher = SHA512.Create(); - var hash = hasher.ComputeHash(data); - return hash; - } - - /// - /// Implements . - /// Specifies that this implementation handles strings. - /// - public bool Handles(Type type) => type == typeof(string); - - /// - /// Implements . - /// - public IEnumerable GetSubtypes(Type type) => Type.EmptyTypes; - - /// - /// Implements . - /// - /// - public MethodInfo GetStaticWriter(Type type) => WriteMappedStringMethodInfo; - - /// - /// Implements . - /// - /// - public MethodInfo GetStaticReader(Type type) => ReadMappedStringMethodInfo; - - private delegate void WriteStringDelegate(Stream stream, string? value); - - private delegate void ReadStringDelegate(Stream stream, out string? value); - - private static readonly MethodInfo WriteMappedStringMethodInfo - = ((WriteStringDelegate) WriteMappedString).Method; - - private static readonly MethodInfo ReadMappedStringMethodInfo - = ((ReadStringDelegate) ReadMappedString).Method; - - private static readonly char[] TrimmableSymbolChars = - { - '.', '\\', '/', ',', '#', '$', '?', '!', '@', '|', '&', '*', '(', ')', '^', '`', '"', '\'', '`', '~', '[', ']', '{', '}', ':', ';', '-' - }; - - /// - /// The shortest a string can be in order to be inserted in the mapping. - /// - /// - /// Strings below a certain length aren't worth compressing. - /// - private const int MinMappedStringSize = 3; - - /// - /// The longest a string can be in order to be inserted in the mapping. - /// - private const int MaxMappedStringSize = 420; - - /// - /// The special value corresponding to a null string in the - /// encoding. - /// - private const int MappedNull = 0; - - /// - /// The special value corresponding to a string which was not mapped. - /// This is followed by the bytes of the unmapped string. - /// - private const int UnmappedString = 1; - - /// - /// The first non-special value, used for encoding mapped strings. - /// - /// - /// Since previous values are taken by and - /// , this value is used to encode - /// mapped strings at an offset - in the encoding, a value - /// >= FirstMappedIndexStart represents the string with - /// mapping of that value - FirstMappedIndexStart. - /// - private const int FirstMappedIndexStart = 2; - - /// - /// Write the encoding of the given string to the stream. - /// - /// The stream to write to. - /// The (possibly null) string to write. - public static void WriteMappedString(Stream stream, string? value) - { - if (!LockMappedStrings) - { - LogSzr.Warning("Performing unlocked string mapping."); - } - - if (value == null) - { - WriteCompressedUnsignedInt(stream, MappedNull); - return; - } - - if (_StringMapping.TryGetValue(value, out var mapping)) - { -#if DEBUG - if (mapping >= _MappedStrings.Count || mapping < 0) - { - throw new InvalidOperationException("A string mapping outside of the mapped string table was encountered."); - } -#endif - WriteCompressedUnsignedInt(stream, (uint) mapping + FirstMappedIndexStart); - //Logger.DebugS("szr", $"Encoded mapped string: {value}"); - return; - } - - // indicate not mapped - WriteCompressedUnsignedInt(stream, UnmappedString); - var buf = Encoding.UTF8.GetBytes(value); - //Logger.DebugS("szr", $"Encoded unmapped string: {value}"); - WriteCompressedUnsignedInt(stream, (uint) buf.Length); - stream.Write(buf); - } - - /// - /// Try to read a string from the given stream.. - /// - /// The stream to read from. - /// The (possibly null) string read. - /// - /// Thrown if the mapping is not locked. - /// - public static void ReadMappedString(Stream stream, out string? value) - { - if (!LockMappedStrings) - { - throw new InvalidOperationException("Not performing unlocked string mapping."); - } - - var mapIndex = ReadCompressedUnsignedInt(stream, out _); - if (mapIndex == MappedNull) - { - value = null; - return; - } - - if (mapIndex == UnmappedString) - { - // not mapped - var length = ReadCompressedUnsignedInt(stream, out _); - var buf = new byte[length]; - stream.Read(buf); - value = Encoding.UTF8.GetString(buf); - //Logger.DebugS("szr", $"Decoded unmapped string: {value}"); - return; - } - - value = _MappedStrings[(int) mapIndex - FirstMappedIndexStart]; - //Logger.DebugS("szr", $"Decoded mapped string: {value}"); - } - -#if ROBUST_SERIALIZER_DISABLE_COMPRESSED_UINTS - public static int WriteCompressedUnsignedInt(Stream stream, uint value) - { - WriteUnsignedInt(stream, value); - return 4; - } - - public static uint ReadCompressedUnsignedInt(Stream stream, out int byteCount) - { - byteCount = 4; - return ReadUnsignedInt(stream); - } -#else - public static int WriteCompressedUnsignedInt(Stream stream, uint value) - { - var length = 1; - while (value >= 0x80) - { - stream.WriteByte((byte) (0x80 | value)); - value >>= 7; - ++length; - } - - stream.WriteByte((byte) value); - return length; - } - - public static uint ReadCompressedUnsignedInt(Stream stream, out int byteCount) - { - byteCount = 0; - var value = 0u; - var shift = 0; - while (stream.CanRead) - { - var current = stream.ReadByte(); - ++byteCount; - if (current == -1) - { - throw new EndOfStreamException(); - } - - value |= (0x7Fu & (byte) current) << shift; - shift += 7; - if ((0x80 & current) == 0) - { - return value; - } - } - - throw new EndOfStreamException(); - } -#endif - - [UsedImplicitly] - public static unsafe void WriteUnsignedInt(Stream stream, uint value) - { - var bytes = MemoryMarshal.AsBytes(new ReadOnlySpan(&value, 1)); - stream.Write(bytes); - } - - [UsedImplicitly] - public static unsafe uint ReadUnsignedInt(Stream stream) - { - uint value; - var bytes = MemoryMarshal.AsBytes(new Span(&value, 1)); - stream.Read(bytes); - return value; - } - - /// - /// See . - /// - public static event Action? ClientHandshakeComplete; - - } - - } - -} diff --git a/Robust.Shared/Serialization/RobustSerializer.cs b/Robust.Shared/Serialization/RobustSerializer.cs index fffacd939..6cf60b983 100644 --- a/Robust.Shared/Serialization/RobustSerializer.cs +++ b/Robust.Shared/Serialization/RobustSerializer.cs @@ -23,6 +23,8 @@ namespace Robust.Shared.Serialization private HashSet _serializableTypes = default!; + private readonly RobustMappedStringSerializer _mappedStringSerializer = new RobustMappedStringSerializer(); + #region Statistics public static long LargestObjectSerializedBytes { get; private set; } @@ -45,7 +47,8 @@ namespace Robust.Shared.Serialization public void Initialize() { - var mappedStringSerializer = new MappedStringSerializer(); + IoCManager.RegisterInstance(_mappedStringSerializer); + var types = _reflectionManager.FindTypesWithAttribute().ToList(); #if !FULL_RELEASE // confirm only shared types are marked for serialization, no client & server only types @@ -65,14 +68,14 @@ namespace Robust.Shared.Serialization var settings = new Settings { - CustomTypeSerializers = new ITypeSerializer[] {mappedStringSerializer} + CustomTypeSerializers = new ITypeSerializer[] {_mappedStringSerializer} }; _serializer = new Serializer(types, settings); _serializableTypes = new HashSet(_serializer.GetTypeMap().Keys); if (_netManager.IsClient) { - MappedStringSerializer.LockMappedStrings = true; + _mappedStringSerializer.LockMappedStrings = true; } else { @@ -80,7 +83,7 @@ namespace Robust.Shared.Serialization var gameAssemblies = _reflectionManager.Assemblies; var robustShared = defaultAssemblies .First(a => a.GetName().Name == "Robust.Shared"); - MappedStringSerializer.AddStrings(robustShared); + _mappedStringSerializer.AddStrings(robustShared); // TODO: Need to add a GetSharedAssemblies method to the reflection manager @@ -88,7 +91,7 @@ namespace Robust.Shared.Serialization .FirstOrDefault(a => a.GetName().Name == "Content.Shared"); if (contentShared != null) { - MappedStringSerializer.AddStrings(contentShared); + _mappedStringSerializer.AddStrings(contentShared); } // TODO: Need to add a GetServerAssemblies method to the reflection manager @@ -97,11 +100,11 @@ namespace Robust.Shared.Serialization .FirstOrDefault(a => a.GetName().Name == "Content.Server"); if (contentServer != null) { - MappedStringSerializer.AddStrings(contentServer); + _mappedStringSerializer.AddStrings(contentServer); } } - MappedStringSerializer.NetworkInitialize(_netManager); + _mappedStringSerializer.NetworkInitialize(_netManager); } public void Serialize(Stream stream, object toSerialize) diff --git a/Robust.Shared/Utility/TypeAbbreviation.cs b/Robust.Shared/Utility/TypeAbbreviation.cs index 719c4765d..e0376b204 100644 --- a/Robust.Shared/Utility/TypeAbbreviation.cs +++ b/Robust.Shared/Utility/TypeAbbreviation.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Text; +using Robust.Shared.IoC; +using Robust.Shared.IoC.Exceptions; using Robust.Shared.Serialization; using YamlDotNet.RepresentationModel; @@ -28,8 +30,6 @@ namespace Robust.Shared.Utility var document = yamlStream.Documents[0]; _abbreviations = ParseAbbreviations((YamlSequenceNode) document.RootNode); - - RobustSerializer.MappedStringSerializer.AddStrings(yamlStream, "(embedded) Robust.Shared.Utility.TypeAbbreviations.yaml"); } /// diff --git a/Robust.UnitTesting/GameControllerDummy.cs b/Robust.UnitTesting/GameControllerDummy.cs index ca51d2bba..aec279e3a 100644 --- a/Robust.UnitTesting/GameControllerDummy.cs +++ b/Robust.UnitTesting/GameControllerDummy.cs @@ -1,6 +1,8 @@ -using Robust.Client; +using System; +using Robust.Client; using Robust.Client.Input; using Robust.Client.Interfaces; +using Robust.Shared.Interfaces.Log; using Robust.Shared.Timing; namespace Robust.UnitTesting @@ -19,7 +21,7 @@ namespace Robust.UnitTesting public bool LoadConfigAndUserData { get; set; } = true; - public bool Startup() + public bool Startup(Func? logHandlerFactory = null) { return true; } diff --git a/Robust.UnitTesting/RobustIntegrationTest.cs b/Robust.UnitTesting/RobustIntegrationTest.cs index 03c06bb44..ff9db6f5c 100644 --- a/Robust.UnitTesting/RobustIntegrationTest.cs +++ b/Robust.UnitTesting/RobustIntegrationTest.cs @@ -197,8 +197,8 @@ namespace Robust.UnitTesting _unhandledException = shutDownMessage.UnhandledException; if (throwOnUnhandled && _unhandledException != null) { - throw new Exception("Waiting instance shut down with unhandled exception", - _unhandledException); + ExceptionDispatchInfo.Capture(_unhandledException).Throw(); + return; } break; @@ -327,7 +327,7 @@ namespace Robust.UnitTesting IoCManager.RegisterInstance(new Mock().Object, true); _options?.InitIoC?.Invoke(); IoCManager.BuildGraph(); - ServerProgram.SetupLogging(); + //ServerProgram.SetupLogging(); ServerProgram.InitReflectionManager(); var server = DependencyCollection.Resolve(); @@ -349,7 +349,7 @@ namespace Robust.UnitTesting .OverrideConVars(_options.CVarOverrides.Select(p => (p.Key, p.Value))); } - if (server.Start()) + if (server.Start(() => new TestLogHandler("SERVER"))) { throw new Exception("Server failed to start."); } @@ -436,7 +436,7 @@ namespace Robust.UnitTesting .OverrideConVars(_options.CVarOverrides.Select(p => (p.Key, p.Value))); }; - client.Startup(); + client.Startup(() => new TestLogHandler("CLIENT")); var gameLoop = new IntegrationGameLoop(DependencyCollection.Resolve(), _fromInstanceWriter, _toInstanceReader); diff --git a/Robust.UnitTesting/TestLogHandler.cs b/Robust.UnitTesting/TestLogHandler.cs new file mode 100644 index 000000000..9a2278171 --- /dev/null +++ b/Robust.UnitTesting/TestLogHandler.cs @@ -0,0 +1,43 @@ +using Robust.Shared.Interfaces.Log; +using System; +using System.Diagnostics; +using System.IO; +using System.Text; +using NUnit.Framework; +using Robust.Shared.Log; +using Robust.Shared.Utility; + +namespace Robust.UnitTesting +{ + + public sealed class TestLogHandler : ILogHandler, IDisposable + { + + private readonly string _prefix; + + private readonly TextWriter _writer; + + private readonly Stopwatch _sw = Stopwatch.StartNew(); + + public TestLogHandler(string prefix) + { + _prefix = prefix; + _writer = TestContext.Out; + _writer.WriteLine($"{_prefix}: Started {DateTime.Now:o}"); + } + + public void Dispose() + { + _writer.Dispose(); + } + + public void Log(in LogMessage message) + { + var name = message.LogLevelToName(); + var seconds = _sw.ElapsedMilliseconds/1000d; + _writer.WriteLine($"{_prefix}: {seconds:F3}s [{name}] {message.SawmillName}: {message.Message}"); + } + + } + +}