Move string serializer to ZStd and BLAKE2b.

Faster and smaller

Robust.Shared/Serialization/RobustMappedStringSerializer.MappedStringDict.cs

@@ -2,13 +2,11 @@
 using System.Buffers;
 using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
 using System.Reflection;
 using System.Reflection.Metadata;
 using System.Reflection.Metadata.Ecma335;
 using System.Runtime.CompilerServices;
-using System.Security.Cryptography;
 using System.Text;
 using NetSerializer;
 using Robust.Shared.Log;
@@ -77,21 +75,21 @@ namespace Robust.Shared.Serialization
 
             public int LoadFromPackage(Stream stream, out byte[] hash)
             {
-                _mappedStrings = ReadStringPackage(stream, out hash).ToArray();
+                _mappedStrings = ReadStringPackage(stream, out hash);
                 _stringMapping = GenMapDict(_mappedStrings);
 
                 return _mappedStrings.Length;
             }
 
-            private static List<string> ReadStringPackage(Stream stream, out byte[] hash)
+            private static string[] ReadStringPackage(Stream stream, out byte[] hash)
             {
-                var list = new List<string>();
                 var buf = ArrayPool<byte>.Shared.Rent(4096);
-                var hasher = IncrementalHash.CreateHash(PackHashAlgo);
-                using var zs = new DeflateStream(stream, CompressionMode.Decompress, true);
-                using var hasherStream = new HasherStream(zs, hasher, true);
+                using var zs = new ZStdDecompressStream(stream, ownStream: false);
+                using var hasherStream = Blake2BHasherStream.CreateReader(zs, ReadOnlySpan<byte>.Empty, 32);
 
                 Primitives.ReadPrimitive(hasherStream, out uint count);
+                var list = new string[count];
+
                 for (var i = 0; i < count; ++i)
                 {
                     Primitives.ReadPrimitive(hasherStream, out uint lu);
@@ -100,14 +98,13 @@ namespace Robust.Shared.Serialization
                     hasherStream.ReadExact(span);
 
                     var str = Encoding.UTF8.GetString(span);
-                    list.Add(str);
+                    list[i] = str;
                 }
 
-                hash = hasher.GetHashAndReset();
+                hash = hasherStream.Finish();
                 return list;
             }
 
-
             /// <summary>
             /// Writes a strings package to a stream.
             /// </summary>
@@ -118,30 +115,27 @@ namespace Robust.Shared.Serialization
                 // ReSharper disable once SuggestVarOrType_Elsewhere
                 Span<byte> buf = stackalloc byte[MaxMappedStringSize];
 
-                var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA512);
+                using var zs = new ZStdCompressStream(stream, ownStream: false);
+                using var hasherStream = Blake2BHasherStream.CreateWriter(zs, ReadOnlySpan<byte>.Empty, 32);
 
-                using (var zs = new DeflateStream(stream, CompressionLevel.Optimal, true))
+                Primitives.WritePrimitive(hasherStream, (uint) strings.Length);
+
+                foreach (var str in strings)
                 {
-                    using var hasherStream = new HasherStream(zs, hasher, true);
-                    Primitives.WritePrimitive(hasherStream, (uint) strings.Length);
+                    DebugTools.Assert(str.Length < MaxMappedStringSize);
 
-                    foreach (var str in strings)
+                    var l = Encoding.UTF8.GetBytes(str, buf);
+
+                    if (l >= MaxMappedStringSize)
                     {
-                        DebugTools.Assert(str.Length < MaxMappedStringSize);
-
-                        var l = Encoding.UTF8.GetBytes(str, buf);
-
-                        if (l >= MaxMappedStringSize)
-                        {
-                            throw new NotImplementedException("Overly long string in strings package.");
-                        }
-
-                        Primitives.WritePrimitive(hasherStream, (uint) l);
-                        hasherStream.Write(buf[..l]);
+                        throw new NotImplementedException("Overly long string in strings package.");
                     }
+
+                    Primitives.WritePrimitive(hasherStream, (uint) l);
+                    hasherStream.Write(buf[..l]);
                 }
 
-                hash = hasher.GetHashAndReset();
+                hash = hasherStream.Finish();
             }
 
 

Robust.Shared/Serialization/RobustMappedStringSerializer.cs

@@ -56,8 +56,6 @@ namespace Robust.Shared.Serialization
             '{', '}', ':', ';', '-'
         };
 
-        private static readonly HashAlgorithmName PackHashAlgo = HashAlgorithmName.SHA512;
-
         /// <summary>
         /// The shortest a string can be in order to be inserted in the mapping.
         /// </summary>

Robust.Shared/Utility/Blake2BHasherStream.cs

@@ -0,0 +1,138 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using SpaceWizards.Sodium;
+
+namespace Robust.Shared.Utility;
+
+internal sealed class Blake2BHasherStream : Stream
+{
+    private readonly bool _reader;
+
+    public readonly int OutputLength;
+    public readonly Stream WrappingStream;
+
+    public CryptoGenericHashBlake2B.State State;
+
+    private Blake2BHasherStream(Stream wrapping, bool reader, ReadOnlySpan<byte> key, int outputLength)
+    {
+        OutputLength = outputLength;
+        WrappingStream = wrapping;
+        _reader = reader;
+
+        CryptoGenericHashBlake2B.Init(ref State, key, outputLength);
+    }
+
+    public byte[] Finish()
+    {
+        var result = new byte[OutputLength];
+
+        CryptoGenericHashBlake2B.Final(ref State, result);
+
+        return result;
+    }
+
+    public static Blake2BHasherStream CreateReader(Stream wrapping, ReadOnlySpan<byte> key, int outputLength)
+    {
+        if (!wrapping.CanRead)
+            throw new ArgumentException("Must pass readable stream.");
+
+        return new Blake2BHasherStream(wrapping, true, key, outputLength);
+    }
+
+    public static Blake2BHasherStream CreateWriter(Stream wrapping, ReadOnlySpan<byte> key, int outputLength)
+    {
+        if (!wrapping.CanWrite)
+            throw new ArgumentException("Must pass writeable stream.");
+
+        return new Blake2BHasherStream(wrapping, false, key, outputLength);
+    }
+
+    public override void Flush()
+    {
+        if (!CanWrite)
+            throw new InvalidOperationException();
+
+        WrappingStream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        if (!CanRead)
+            throw new InvalidOperationException();
+
+        var read = WrappingStream.Read(buffer, offset, count);
+
+        if (read > 0)
+            CryptoGenericHashBlake2B.Update(ref State, buffer.AsSpan(offset, read));
+
+        return read;
+    }
+
+    public override int Read(Span<byte> buffer)
+    {
+        if (!CanRead)
+            throw new InvalidOperationException();
+
+        var read = WrappingStream.Read(buffer);
+
+        if (read > 0)
+            CryptoGenericHashBlake2B.Update(ref State, buffer[..read]);
+
+        return read;
+    }
+
+    public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+    {
+        if (!CanRead)
+            throw new InvalidOperationException();
+
+        var read = await WrappingStream.ReadAsync(buffer, cancellationToken);
+
+        if (read > 0)
+            CryptoGenericHashBlake2B.Update(ref State, buffer[..read].Span);
+
+        return read;
+    }
+
+    public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+
+    public override void SetLength(long value) => throw new NotSupportedException();
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        if (!CanWrite)
+            throw new InvalidOperationException();
+
+        WrappingStream.Write(buffer, offset, count);
+        CryptoGenericHashBlake2B.Update(ref State, buffer.AsSpan(offset, count));
+    }
+
+    public override void Write(ReadOnlySpan<byte> buffer)
+    {
+        if (!CanWrite)
+            throw new InvalidOperationException();
+
+        WrappingStream.Write(buffer);
+        CryptoGenericHashBlake2B.Update(ref State, buffer);
+    }
+
+    public override void WriteByte(byte value)
+    {
+        Span<byte> buf = stackalloc byte[1];
+        buf[0] = value;
+        Write(buf);
+    }
+
+    public override bool CanRead => _reader;
+    public override bool CanSeek => false;
+    public override bool CanWrite => !_reader;
+    public override long Length => throw new NotSupportedException();
+
+    public override long Position
+    {
+        get => throw new NotSupportedException();
+        set => throw new NotSupportedException();
+    }
+}