ACZ manifest delta downloads (#2698)

Robust.Server/ServerStatus/IStatusHandlerContext.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Net;
 using System.Net.Http;
 using System.Threading.Tasks;
@@ -11,10 +12,18 @@ namespace Robust.Server.ServerStatus
     {
         HttpMethod RequestMethod { get; }
         IPEndPoint RemoteEndPoint { get; }
+
+        /// <summary>
+        /// Stream that reads the request body data,
+        /// </summary>
+        Stream RequestBody { get; }
         Uri Url { get; }
         bool IsGetLike { get; }
         IReadOnlyDictionary<string, StringValues> RequestHeaders { get; }
 
+        IDictionary<string, string> ResponseHeaders { get; }
+        bool KeepAlive { get; set; }
+
         [Obsolete("Use async versions instead")]
         T? RequestBodyJson<T>();
         Task<T?> RequestBodyJsonAsync<T>();
@@ -43,6 +52,8 @@ namespace Robust.Server.ServerStatus
             int code = 200,
             string contentType = "text/plain");
 
+        Task RespondNoContentAsync();
+
         Task RespondAsync(
             string text,
             HttpStatusCode code = HttpStatusCode.OK,
@@ -72,5 +83,7 @@ namespace Robust.Server.ServerStatus
         void RespondJson(object jsonData, HttpStatusCode code = HttpStatusCode.OK);
 
         Task RespondJsonAsync(object jsonData, HttpStatusCode code = HttpStatusCode.OK);
+
+        Task<Stream> RespondStreamAsync(HttpStatusCode code = HttpStatusCode.OK);
     }
 }

Robust.Server/ServerStatus/StatusHost.Acz.cs

@@ -0,0 +1,654 @@
+using System;
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Collections;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Security.Cryptography;
+using Robust.Shared;
+using Robust.Shared.ContentPack;
+using Robust.Shared.Utility;
+using Robust.Shared.Utility.Collections;
+using SharpZstd.Interop;
+using SpaceWizards.Sodium;
+
+namespace Robust.Server.ServerStatus
+{
+    // Contains primary logic for ACZ (Automatic Client Zip)
+    // This entails the following:
+    // * Automatic generation of client zip on development servers.
+    // * Loading of pre-built client zip on release servers. ("Hybrid ACZ")
+    // * Distribution of the above two via status host, to facilitate easier server setup.
+    // * Manifest-based download system from the above.
+
+    internal sealed partial class StatusHost
+    {
+        // Lock used while working on the ACZ.
+        private readonly SemaphoreSlim _aczLock = new(1, 1);
+
+        // If an attempt has been made to prepare the ACZ.
+        private bool _aczPrepareAttempted = false;
+
+        // Automatic Client Zip
+        private AutomaticClientZipInfo? _aczPrepared;
+
+        private (string binFolder, string[] assemblies)? _aczInfo;
+
+        private void AddAczHandlers()
+        {
+            AddHandler(HandleAutomaticClientZip);
+            AddHandler(HandleAczManifest);
+            AddHandler(HandleAczManifestDownload);
+        }
+
+        private void InitAcz()
+        {
+            _cfg.OnValueChanged(CVars.AczStreamCompress, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczStreamCompressLevel, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczBlobCompress, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczBlobCompressLevel, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczBlobCompressSaveThreshold, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczManifestCompress, _ => InvalidateAcz());
+            _cfg.OnValueChanged(CVars.AczManifestCompressLevel, _ => InvalidateAcz());
+        }
+
+        private void InvalidateAcz()
+        {
+            using var _ = _aczLock.WaitGuard();
+
+            if (_aczPrepared == null)
+                return;
+
+            _aczSawmill.Info("ACZ CVars changed, invalidating ACZ data.");
+
+            _aczPrepared = null;
+            _aczPrepareAttempted = false;
+        }
+
+        private async Task<bool> HandleAutomaticClientZip(IStatusHandlerContext context)
+        {
+            if (!context.IsGetLike || context.Url!.AbsolutePath != "/client.zip")
+            {
+                return false;
+            }
+
+            if (!string.IsNullOrEmpty(_cfg.GetCVar(CVars.BuildDownloadUrl)))
+            {
+                await context.RespondAsync("This server has a build download URL.", HttpStatusCode.NotFound);
+                return true;
+            }
+
+            var result = await PrepareACZ();
+            if (result == null)
+            {
+                await context.RespondAsync("Automatic Client Zip was not preparable.",
+                    HttpStatusCode.InternalServerError);
+                return true;
+            }
+
+            await context.RespondAsync(result.ZipData, HttpStatusCode.OK, "application/zip");
+            return true;
+        }
+
+        private async Task<bool> HandleAczManifest(IStatusHandlerContext context)
+        {
+            if (!context.IsGetLike || context.Url!.AbsolutePath != "/manifest.txt")
+                return false;
+
+            if (!string.IsNullOrEmpty(_cfg.GetCVar(CVars.BuildManifestUrl)))
+            {
+                await context.RespondAsync("This server has a build manifest URL.", HttpStatusCode.NotFound);
+                return true;
+            }
+
+            var result = await PrepareACZ();
+            if (result == null)
+            {
+                await context.RespondAsync("Automatic Client Zip was not preparable.",
+                    HttpStatusCode.InternalServerError);
+                return true;
+            }
+
+            if (RequestWantsZStd(context) && result.ManifestCompressed)
+            {
+                context.ResponseHeaders.Add("Content-Encoding", "zstd");
+
+                await context.RespondAsync(result.ManifestData, HttpStatusCode.OK);
+            }
+            else
+            {
+                if (result.ManifestCompressed)
+                {
+                    // Manifest is compressed in-memory but client didn't want it compressed.
+                    // Have to decompress ourselves.
+
+                    var ms = new MemoryStream(result.ManifestData);
+
+                    await using var stream = await context.RespondStreamAsync();
+                    await using var decompressStream = new ZStdDecompressStream(ms);
+
+                    await decompressStream.CopyToAsync(stream);
+                }
+                else
+                {
+                    await context.RespondAsync(result.ManifestData, HttpStatusCode.OK);
+                }
+            }
+
+            return true;
+        }
+
+        private async Task<bool> HandleAczManifestDownload(IStatusHandlerContext context)
+        {
+            if (context.Url.AbsolutePath != "/download")
+                return false;
+
+            if (context.RequestHeaders.ContainsKey("Content-Type")
+                && context.RequestHeaders["Content-Type"] != "application/octet-stream")
+            {
+                await context.RespondAsync(
+                    "Must specify application/octet-stream Content-Type",
+                    HttpStatusCode.BadRequest);
+            }
+
+            if (!string.IsNullOrEmpty(_cfg.GetCVar(CVars.BuildManifestUrl)))
+            {
+                await context.RespondAsync("This server has a build manifest URL.", HttpStatusCode.NotFound);
+                return true;
+            }
+
+            // HTTP OPTIONS
+            if (context.RequestMethod == HttpMethod.Options)
+            {
+                context.ResponseHeaders["X-Robust-Download-Min-Protocol"] = "1";
+                context.ResponseHeaders["X-Robust-Download-Max-Protocol"] = "1";
+                await context.RespondNoContentAsync();
+                return true;
+            }
+
+            if (context.RequestMethod != HttpMethod.Post)
+                return false;
+
+            var aczInfo = await PrepareACZ();
+            if (aczInfo == null)
+            {
+                await context.RespondAsync("Automatic Client Zip was not preparable.",
+                    HttpStatusCode.InternalServerError);
+                return true;
+            }
+
+            // HTTP POST: main handling system.
+
+            // Verify version request header.
+            // Right now only one version ("1") exists, so...
+
+            // Request body not yet read, don't allow keepalive.
+            context.KeepAlive = false;
+            if (!context.RequestHeaders.TryGetValue("X-Robust-Download-Protocol", out var versionHeader)
+                || versionHeader.Count != 1
+                || !Parse.TryInt32(versionHeader[0], out var version))
+            {
+                await context.RespondAsync("Expected single X-Robust-Download-Protocol header",
+                    HttpStatusCode.BadRequest);
+                return true;
+            }
+
+            if (version != 1)
+            {
+                await context.RespondAsync("Unsupported download protocol version", HttpStatusCode.NotImplemented);
+                return true;
+            }
+
+            var fileCount = aczInfo.ManifestEntries.Length;
+
+            var requestBufSize = fileCount * 4;
+            var pool = ArrayPool<byte>.Shared.Rent(requestBufSize);
+            using var poolGuard = ArrayPool<byte>.Shared.ReturnGuard(pool);
+            var buffer = new MemoryStream(
+                pool,
+                0, requestBufSize,
+                writable: true,
+                publiclyVisible: true);
+
+            try
+            {
+                await context.RequestBody.CopyToAsync(buffer);
+            }
+            catch (NotSupportedException)
+            {
+                // Thrown by memory stream if full.
+                await context.RespondAsync("Request too large", HttpStatusCode.RequestEntityTooLarge);
+                return true;
+            }
+
+            // Request body read, allow keepalive again.
+            context.KeepAlive = true;
+
+            // Request body read. Validate it.
+            // Do not allow out-of-bounds files or duplicate requests.
+
+            var buf = pool.AsMemory(0, (int)buffer.Position);
+
+            var manifestLength = aczInfo.ManifestEntries.Length;
+            var bits = new BitArray(manifestLength);
+            var offset = 0;
+            while (offset < buf.Length)
+            {
+                var index = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(offset, 4).Span);
+
+                if (index < 0 || index >= manifestLength)
+                {
+                    await context.RespondAsync("Out of bounds manifest index", HttpStatusCode.BadRequest);
+                    return true;
+                }
+
+                if (bits[index])
+                {
+                    await context.RespondAsync("Cannot request file twice", HttpStatusCode.BadRequest);
+                    return true;
+                }
+
+                bits[index] = true;
+
+                offset += 4;
+            }
+
+            // There is a theoretical tiny race condition here where the main thread may change these parameters
+            // between us acquiring the ACZ info above and reading them here.
+            // The worst that could happen here is that the stream is either double-compressed or not compressed at all,
+            // So I am not too worried and am just gonna leave it as-is.
+
+            var cVarStreamCompression = _cfg.GetCVar(CVars.AczStreamCompress);
+            var cVarStreamCompressionLevel = _cfg.GetCVar(CVars.AczStreamCompressLevel);
+
+            // Only do zstd stream compression if the client asks for it and we have it enabled.
+            var doStreamCompression = RequestWantsZStd(context)
+                                      && cVarStreamCompression;
+
+            if (doStreamCompression)
+                context.ResponseHeaders["Content-Encoding"] = "zstd";
+
+            var outStream = await context.RespondStreamAsync();
+
+            if (doStreamCompression)
+            {
+                var zStdCompressStream = new ZStdCompressStream(outStream);
+                zStdCompressStream.Context.SetParameter(
+                    ZSTD_cParameter.ZSTD_c_compressionLevel,
+                    cVarStreamCompressionLevel);
+
+                outStream = zStdCompressStream;
+            }
+
+            var preCompressed = aczInfo.PreCompressed;
+
+            var fileHeaderSize = 4;
+            if (preCompressed)
+                fileHeaderSize += 4;
+
+            var fileHeader = new byte[fileHeaderSize];
+
+            await using (outStream)
+            {
+                var streamHeader = new byte[4];
+                DownloadStreamHeaderFlags streamHeaderFlags = 0;
+                if (preCompressed)
+                    streamHeaderFlags |= DownloadStreamHeaderFlags.PreCompressed;
+
+                BinaryPrimitives.WriteInt32LittleEndian(streamHeader, (int)streamHeaderFlags);
+
+                await outStream.WriteAsync(streamHeader);
+
+                offset = 0;
+                while (offset < buf.Length)
+                {
+                    var index = BinaryPrimitives.ReadInt32LittleEndian(buf.Slice(offset, 4).Span);
+
+                    var (blobLength, dataOffset, dataLength) = aczInfo.ManifestEntries[index];
+
+                    // _aczSawmill.Debug($"{index:D5}: {blobLength:D8} {dataOffset:D8} {dataLength:D8}");
+
+                    BinaryPrimitives.WriteInt32LittleEndian(fileHeader, blobLength);
+
+                    if (preCompressed)
+                        BinaryPrimitives.WriteInt32LittleEndian(fileHeader.AsSpan(4, 4), dataLength);
+
+                    var writeLength = dataLength == 0 ? blobLength : dataLength;
+
+                    await outStream.WriteAsync(fileHeader);
+
+                    await outStream.WriteAsync(aczInfo.ManifestBlobData.AsMemory(dataOffset, writeLength));
+
+                    offset += 4;
+                }
+            }
+
+            return true;
+        }
+
+        private static bool RequestWantsZStd(IStatusHandlerContext context)
+        {
+            // Yeah this isn't a good parser for Accept-Encoding but who cares.
+            return context.RequestHeaders.TryGetValue("Accept-Encoding", out var ac) && ac[0].Contains("zstd");
+        }
+
+        // Only call this if the download URL is not available!
+        private async Task<AutomaticClientZipInfo?> PrepareACZ()
+        {
+            // Take the ACZ lock asynchronously
+            await _aczLock.WaitAsync();
+            try
+            {
+                // Setting this now ensures that it won't fail repeatedly on exceptions/etc.
+                if (_aczPrepareAttempted)
+                    return _aczPrepared;
+
+                _aczPrepareAttempted = true;
+                // ACZ hasn't been prepared, prepare it
+                try
+                {
+                    // Run actual ACZ generation via Task.Run because it's synchronous
+                    var maybeData = await Task.Run(PrepareACZInnards);
+                    if (maybeData == null)
+                    {
+                        _aczSawmill.Error("StatusHost PrepareACZ failed (server will not be usable from launcher!)");
+                        return null;
+                    }
+
+                    _aczPrepared = maybeData;
+                    return maybeData;
+                }
+                catch (Exception e)
+                {
+                    _aczSawmill.Error(
+                        $"Exception in StatusHost PrepareACZ (server will not be usable from launcher!): {e}");
+                    return null;
+                }
+            }
+            finally
+            {
+                _aczLock.Release();
+            }
+        }
+
+        // -- All methods from this point forward do not access the ACZ global state --
+
+        private AutomaticClientZipInfo? PrepareACZInnards()
+        {
+            _aczSawmill.Info("Preparing ACZ...");
+            // All of these should Info on success and Error on null-return failure
+            var zipData = PrepareACZViaFile() ?? PrepareACZViaMagic();
+            if (zipData == null)
+                return null;
+
+            var streamCompression = _cfg.GetCVar(CVars.AczStreamCompress);
+            var blobCompress = _cfg.GetCVar(CVars.AczBlobCompress);
+            var blobCompressLevel = _cfg.GetCVar(CVars.AczBlobCompressLevel);
+            var blobCompressSaveThresh = _cfg.GetCVar(CVars.AczBlobCompressSaveThreshold);
+            var manifestCompress = _cfg.GetCVar(CVars.AczManifestCompress);
+            var manifestCompressLevel = _cfg.GetCVar(CVars.AczManifestCompressLevel);
+
+            // Stream compression disables individual compression.
+            blobCompress &= !streamCompression;
+
+            _aczSawmill.Debug("Making ACZ manifest...");
+            var dataHash = Convert.ToHexString(SHA256.HashData(zipData));
+
+            using var zip = OpenZip(zipData);
+            var (manifestData, manifestEntries, manifestBlobData) = CalcManifestData(
+                zip,
+                blobCompress,
+                blobCompressLevel,
+                blobCompressSaveThresh);
+
+            var manifestHash = CryptoGenericHashBlake2B.Hash(32, manifestData, ReadOnlySpan<byte>.Empty);
+            var manifestHashString = Convert.ToHexString(manifestHash);
+
+            _aczSawmill.Debug("ACZ Manifest hash: {ManifestHash}", manifestHashString);
+
+            if (manifestCompress)
+            {
+                _aczSawmill.Debug("Compressing ACZ manifest at level {ManifestCompressLevel}", manifestCompressLevel);
+
+                var beforeSize = manifestData.Length;
+                var compressBuffer = (int) Zstd.ZSTD_COMPRESSBOUND((nuint) manifestData.Length);
+                var compressed = ArrayPool<byte>.Shared.Rent(compressBuffer);
+
+                var size = ZStd.Compress(compressed, manifestData, manifestCompressLevel);
+
+                manifestData = compressed[..size];
+
+                ArrayPool<byte>.Shared.Return(compressed);
+
+                _aczSawmill.Debug(
+                    "ACZ manifest compression: {ManifestSize} -> {ManifestSizeCompressed} ({ManifestSizeRatio} ratio)",
+                    beforeSize, manifestData.Length, manifestData.Length / (float) beforeSize);
+            }
+
+            return new AutomaticClientZipInfo(
+                zipData,
+                dataHash,
+                manifestData,
+                manifestCompress,
+                manifestHashString,
+                manifestBlobData,
+                manifestEntries,
+                blobCompress);
+        }
+
+        private static (byte[] manifestContent, AczManifestEntry[] manifestEntries, byte[] blobData)
+            CalcManifestData(
+                ZipArchive zip,
+                bool blobCompress,
+                int blobCompressLevel,
+                int blobCompressSaveThresh)
+        {
+            var blobData = new MemoryStream();
+            ZStdCompressStream? compressStream = null;
+            if (blobCompress)
+            {
+                var zStdCompressStream = new ZStdCompressStream(blobData);
+                zStdCompressStream.Context.SetParameter(
+                    ZSTD_cParameter.ZSTD_c_compressionLevel,
+                    blobCompressLevel);
+
+                compressStream = zStdCompressStream;
+            }
+
+            try
+            {
+                var decompressBuffer = ArrayPool<byte>.Shared.Rent(1024 * 1024);
+                Span<byte> entryHash = stackalloc byte[256 / 8];
+
+                var manifestStream = new MemoryStream();
+                using var manifestWriter = new StreamWriter(manifestStream, EncodingHelpers.UTF8);
+                manifestWriter.Write("Robust Content Manifest 1\n");
+
+                var manifestEntries = new ValueList<AczManifestEntry>();
+
+                foreach (var entry in zip.Entries.OrderBy(e => e.FullName, StringComparer.Ordinal))
+                {
+                    // Ignore directory entries.
+                    if (entry.Name == "")
+                        continue;
+
+                    var length = (int)entry.Length;
+                    var startPos = (int)blobData.Position;
+
+                    BufferHelpers.EnsurePooledBuffer(ref decompressBuffer, ArrayPool<byte>.Shared, length);
+                    var data = decompressBuffer.AsSpan(0, length);
+
+                    using (var stream = entry.Open())
+                    {
+                        stream.ReadExact(data);
+                    }
+
+                    // Calculate hash.
+                    CryptoGenericHashBlake2B.Hash(entryHash, data, ReadOnlySpan<byte>.Empty);
+
+                    // Set to 0 to indicate not compressed.
+                    int dataLength;
+
+                    // Try compression if enabled.
+                    if (blobCompress)
+                    {
+                        // Actually compress.
+                        compressStream!.Write(data);
+                        compressStream.FlushEnd();
+
+                        // See if compression was worth it.
+                        var endPos = (int)blobData.Position;
+                        var compressedSize = endPos - startPos;
+                        if (compressedSize + blobCompressSaveThresh < length)
+                        {
+                            dataLength = compressedSize;
+                        }
+                        else
+                        {
+                            // Compression not worth it, just send an uncompressed blob instead.
+                            blobData.Position = startPos;
+                            blobData.Write(data);
+                            dataLength = 0;
+                        }
+                    }
+                    else
+                    {
+                        // No compression, just write.
+                        blobData.Write(data);
+                        dataLength = 0;
+                    }
+
+                    manifestWriter.Write($"{Convert.ToHexString(entryHash)} {entry.FullName}\n");
+
+                    manifestEntries.Add(new AczManifestEntry(length, startPos, dataLength));
+                }
+
+                manifestWriter.Flush();
+
+                ArrayPool<byte>.Shared.Return(decompressBuffer);
+
+                return (manifestStream.ToArray(), manifestEntries.ToArray(), blobData.ToArray());
+            }
+            finally
+            {
+                compressStream?.Dispose();
+            }
+        }
+
+        private static ZipArchive OpenZip(byte[] data)
+        {
+            var ms = new MemoryStream(data, false);
+            return new ZipArchive(ms, ZipArchiveMode.Read, leaveOpen: false);
+        }
+
+        private byte[]? PrepareACZViaFile()
+        {
+            var path = PathHelpers.ExecutableRelativeFile("Content.Client.zip");
+            if (!File.Exists(path)) return null;
+            _aczSawmill.Info($"StatusHost found client zip: {path}");
+            return File.ReadAllBytes(path);
+        }
+
+        private byte[]? PrepareACZViaMagic()
+        {
+            var sw = Stopwatch.StartNew();
+
+            var (binFolderPath, assemblyNames) =
+                _aczInfo ?? ("Content.Client", new[] { "Content.Client", "Content.Shared" });
+
+            var outStream = new MemoryStream();
+            var archive = new ZipArchive(outStream, ZipArchiveMode.Create);
+
+            foreach (var assemblyName in assemblyNames)
+            {
+                AttemptPullFromDisk($"Assemblies/{assemblyName}.dll", $"../../bin/{binFolderPath}/{assemblyName}.dll");
+                AttemptPullFromDisk($"Assemblies/{assemblyName}.pdb", $"../../bin/{binFolderPath}/{assemblyName}.pdb");
+            }
+
+            var prefix = PathHelpers.ExecutableRelativeFile("../../Resources");
+            foreach (var path in PathHelpers.GetFiles(prefix))
+            {
+                var relPath = Path.GetRelativePath(prefix, path);
+                if (OperatingSystem.IsWindows())
+                    relPath = relPath.Replace('\\', '/');
+                AttemptPullFromDisk(relPath, path);
+            }
+
+            archive.Dispose();
+            _aczSawmill.Info("StatusHost synthesized client zip in {Elapsed} ms!", sw.ElapsedMilliseconds);
+            return outStream.ToArray();
+
+            void AttemptPullFromDisk(string pathTo, string pathFrom)
+            {
+                // _aczSawmill.Debug($"StatusHost PrepareACZMagic: {pathFrom} -> {pathTo}");
+                var res = PathHelpers.ExecutableRelativeFile(pathFrom);
+                if (!File.Exists(res))
+                    return;
+
+                var entry = archive.CreateEntry(pathTo);
+
+                using var file = File.OpenRead(res);
+                using var entryStream = entry.Open();
+
+                file.CopyTo(entryStream);
+            }
+        }
+
+        public void SetAczInfo(string clientBinFolder, string[] clientAssemblyNames)
+        {
+            _aczLock.Wait();
+            try
+            {
+                if (_aczPrepared != null)
+                    throw new InvalidOperationException("ACZ already prepared");
+
+                _aczInfo = (clientBinFolder, clientAssemblyNames);
+            }
+            finally
+            {
+                _aczLock.Release();
+            }
+        }
+
+        [Flags]
+        private enum DownloadStreamHeaderFlags
+        {
+            None = 0,
+
+            /// <summary>
+            /// If this flag is set on the download stream, individual files have been pre-compressed by the server.
+            /// This means each file has a compression header, and the launcher should not attempt to compress files itself.
+            /// </summary>
+            PreCompressed = 1 << 0
+        }
+
+        /// <param name="ZipData">Byte array containing the raw zip file data.</param>
+        /// <param name="ZipHash">Hex SHA256 hash of <see cref="ZipData"/>.</param>
+        /// <param name="ManifestData">Data for the content manifest</param>
+        /// <param name="ManifestHash">Hex BLAKE2B 256-bit hash of <see cref="ManifestData"/>.</param>
+        /// <param name="ManifestEntries">Manifest -> zip entry map.</param>
+        internal sealed record AutomaticClientZipInfo(
+            byte[] ZipData,
+            string ZipHash,
+            byte[] ManifestData,
+            bool ManifestCompressed,
+            string ManifestHash,
+            byte[] ManifestBlobData,
+            AczManifestEntry[] ManifestEntries,
+            bool PreCompressed);
+
+        /// <param name="BlobLength">Length of the uncompressed blob.</param>
+        /// <param name="DataOffset">Offset into <see cref="AutomaticClientZipInfo.ManifestBlobData"/> that this blob's (possibly compressed) data starts at.</param>
+        /// <param name="DataLength">
+        /// Length in <see cref="AutomaticClientZipInfo.ManifestBlobData"/> for this blob's (possibly compressed) data.
+        /// If this is zero, it means the file is not stored uncompressed and you should use <see cref="BlobLength"/>.
+        /// </param>
+        internal record struct AczManifestEntry(int BlobLength, int DataOffset, int DataLength);
+    }
+}

Robust.Server/ServerStatus/StatusHost.ClientZip.cs

@@ -1,177 +0,0 @@
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using System.Collections.Generic;
-using System.IO;
-using System.IO.Compression;
-using System.Net;
-using System.Security.Cryptography;
-using Robust.Shared;
-using Robust.Shared.ContentPack;
-
-namespace Robust.Server.ServerStatus
-{
-
-    internal sealed partial class StatusHost
-    {
-        // Lock used while working on the ACZ.
-        private readonly SemaphoreSlim _aczLock = new(1, 1);
-        // If an attempt has been made to prepare the ACZ.
-        private bool _aczPrepareAttempted = false;
-        // Automatic Client Zip
-        private AutomaticClientZipInfo? _aczPrepared;
-
-        private (string binFolder, string[] assemblies)? _aczInfo;
-
-        private async Task<bool> HandleAutomaticClientZip(IStatusHandlerContext context)
-        {
-            if (!context.IsGetLike || context.Url!.AbsolutePath != "/client.zip")
-            {
-                return false;
-            }
-
-            if (!string.IsNullOrEmpty(_configurationManager.GetCVar(CVars.BuildDownloadUrl)))
-            {
-                await context.RespondAsync("This server has a build download URL.", HttpStatusCode.NotFound);
-                return true;
-            }
-
-            var result = await PrepareACZ();
-            if (result == null)
-            {
-                await context.RespondAsync("Automatic Client Zip was not preparable.", HttpStatusCode.InternalServerError);
-                return true;
-            }
-
-            await context.RespondAsync(result.Value.Data, HttpStatusCode.OK, "application/zip");
-            return true;
-        }
-
-        // Only call this if the download URL is not available!
-        private async Task<AutomaticClientZipInfo?> PrepareACZ()
-        {
-            // Take the ACZ lock asynchronously
-            await _aczLock.WaitAsync();
-            try
-            {
-                // Setting this now ensures that it won't fail repeatedly on exceptions/etc.
-                if (_aczPrepareAttempted) return _aczPrepared;
-                _aczPrepareAttempted = true;
-                // ACZ hasn't been prepared, prepare it
-                byte[] data;
-                try
-                {
-                    // Run actual ACZ generation via Task.Run because it's synchronous
-                    var maybeData = await Task.Run(PrepareACZInnards);
-                    if (maybeData == null)
-                    {
-                        _httpSawmill.Error("StatusHost PrepareACZ failed (server will not be usable from launcher!)");
-                        return null;
-                    }
-                    data = maybeData;
-                }
-                catch (Exception e)
-                {
-                    _httpSawmill.Error($"Exception in StatusHost PrepareACZ (server will not be usable from launcher!): {e}");
-                    return null;
-                }
-                _aczPrepared = new AutomaticClientZipInfo(data);
-                return _aczPrepared;
-            }
-            finally
-            {
-                _aczLock.Release();
-            }
-        }
-
-        // -- All methods from this point forward do not access the ACZ global state --
-
-        private byte[]? PrepareACZInnards()
-        {
-            // All of these should Info on success and Error on null-return failure
-            return PrepareACZViaFile() ?? PrepareACZViaMagic();
-        }
-
-        private byte[]? PrepareACZViaFile()
-        {
-            var path = PathHelpers.ExecutableRelativeFile("Content.Client.zip");
-            if (!File.Exists(path)) return null;
-            _httpSawmill.Info($"StatusHost found client zip: {path}");
-            return File.ReadAllBytes(path);
-        }
-
-        private byte[]? PrepareACZViaMagic()
-        {
-            var paths = new Dictionary<string, byte[]>();
-            bool AttemptPullFromDisk(string pathTo, string pathFrom)
-            {
-                // _httpSawmill.Debug($"StatusHost PrepareACZMagic: {pathFrom} -> {pathTo}");
-                var res = PathHelpers.ExecutableRelativeFile(pathFrom);
-                if (!File.Exists(res)) return false;
-                paths[pathTo] = File.ReadAllBytes(res);
-                return true;
-            }
-
-            var (binFolderPath, assemblyNames) =
-                _aczInfo ?? ("Content.Client", new[] { "Content.Client", "Content.Shared" });
-
-            foreach (var assemblyName in assemblyNames)
-            {
-                AttemptPullFromDisk($"Assemblies/{assemblyName}.dll", $"../../bin/{binFolderPath}/{assemblyName}.dll");
-                AttemptPullFromDisk($"Assemblies/{assemblyName}.pdb", $"../../bin/{binFolderPath}/{assemblyName}.pdb");
-            }
-
-            var prefix = PathHelpers.ExecutableRelativeFile("../../Resources");
-            foreach (var path in PathHelpers.GetFiles(prefix))
-            {
-                var relPath = Path.GetRelativePath(prefix, path);
-                if (OperatingSystem.IsWindows())
-                    relPath = relPath.Replace('\\', '/');
-                AttemptPullFromDisk(relPath, path);
-            }
-
-            var outStream = new MemoryStream();
-            var archive = new ZipArchive(outStream, ZipArchiveMode.Create);
-            foreach (var kvp in paths)
-            {
-                var entry = archive.CreateEntry(kvp.Key);
-                using (var entryStream = entry.Open())
-                {
-                    entryStream.Write(kvp.Value);
-                }
-            }
-            archive.Dispose();
-            _httpSawmill.Info($"StatusHost synthesized client zip!");
-            return outStream.ToArray();
-        }
-
-        public void SetAczInfo(string clientBinFolder, string[] clientAssemblyNames)
-        {
-            _aczLock.Wait();
-            try
-            {
-                if (_aczPrepared != null)
-                    throw new InvalidOperationException("ACZ already prepared");
-
-                _aczInfo = (clientBinFolder, clientAssemblyNames);
-            }
-            finally
-            {
-                _aczLock.Release();
-            }
-        }
-    }
-
-    internal struct AutomaticClientZipInfo
-    {
-        public readonly byte[] Data;
-        public readonly string Hash;
-
-        public AutomaticClientZipInfo(byte[] data)
-        {
-            Data = data;
-            using var sha = SHA256.Create();
-            Hash = Convert.ToHexString(sha.ComputeHash(data));
-        }
-    }
-}

Robust.Server/ServerStatus/StatusHost.Handlers.cs

@@ -15,7 +15,7 @@ namespace Robust.Server.ServerStatus
             AddHandler(HandleTeapot);
             AddHandler(HandleStatus);
             AddHandler(HandleInfo);
-            AddHandler(HandleAutomaticClientZip);
+            AddAczHandlers();
         }
 
         private static async Task<bool> HandleTeapot(IStatusHandlerContext context)
@@ -58,7 +58,7 @@ namespace Robust.Server.ServerStatus
                 return false;
             }
 
-            var downloadUrl = _configurationManager.GetCVar(CVars.BuildDownloadUrl);
+            var downloadUrl = _cfg.GetCVar(CVars.BuildDownloadUrl);
 
             JsonObject? buildInfo;
 
@@ -68,20 +68,7 @@ namespace Robust.Server.ServerStatus
             }
             else
             {
-                var hash = _configurationManager.GetCVar(CVars.BuildHash);
-                if (hash == "")
-                {
-                    hash = null;
-                }
-
-                buildInfo = new JsonObject
-                {
-                    ["engine_version"] = _configurationManager.GetCVar(CVars.BuildEngineVersion),
-                    ["fork_id"] = _configurationManager.GetCVar(CVars.BuildForkId),
-                    ["version"] = _configurationManager.GetCVar(CVars.BuildVersion),
-                    ["download_url"] = downloadUrl,
-                    ["hash"] = hash,
-                };
+                buildInfo = GetExternalBuildInfo();
             }
 
             var authInfo = new JsonObject
@@ -94,7 +81,7 @@ namespace Robust.Server.ServerStatus
 
             var jObject = new JsonObject
             {
-                ["connect_address"] = _configurationManager.GetCVar(CVars.StatusConnectAddress),
+                ["connect_address"] = _cfg.GetCVar(CVars.StatusConnectAddress),
                 ["auth"] = authInfo,
                 ["build"] = buildInfo
             };
@@ -106,6 +93,54 @@ namespace Robust.Server.ServerStatus
             return true;
         }
 
+        private JsonObject GetExternalBuildInfo()
+        {
+            var zipHash = _cfg.GetCVar(CVars.BuildHash);
+            var manifestHash = _cfg.GetCVar(CVars.BuildManifestHash);
+            var forkId = _cfg.GetCVar(CVars.BuildForkId);
+            var forkVersion = _cfg.GetCVar(CVars.BuildVersion);
+
+            var manifestDownloadUrl = Interpolate(_cfg.GetCVar(CVars.BuildManifestDownloadUrl));
+            var manifestUrl = Interpolate(_cfg.GetCVar(CVars.BuildManifestUrl));
+            var downloadUrl = Interpolate(_cfg.GetCVar(CVars.BuildDownloadUrl));
+
+            if (zipHash == "")
+                zipHash = null;
+
+            if (manifestHash == "")
+                manifestHash = null;
+
+            if (manifestDownloadUrl == "")
+                manifestDownloadUrl = null;
+
+            if (manifestUrl == "")
+                manifestUrl = null;
+
+            return new JsonObject
+            {
+                ["engine_version"] = _cfg.GetCVar(CVars.BuildEngineVersion),
+                ["fork_id"] = forkId,
+                ["version"] = forkVersion,
+                ["download_url"] = downloadUrl,
+                ["hash"] = zipHash,
+                ["acz"] = false,
+                ["manifest_download_url"] = manifestDownloadUrl,
+                ["manifest_url"] = manifestUrl,
+                ["manifest_hash"] = manifestHash
+            };
+
+            string? Interpolate(string? value)
+            {
+                // Can't tell if splitting the ?. like this is more cursed than
+                // failing to align due to putting the full ?. on the next line
+                return value?
+                    .Replace("{FORK_VERSION}", forkVersion)
+                    .Replace("{FORK_ID}", forkId)
+                    .Replace("{MANIFEST_HASH}", manifestHash)
+                    .Replace("{ZIP_HASH}", zipHash);
+            }
+        }
+
         private async Task<JsonObject?> PrepareACZBuildInfo()
         {
             var acz = await PrepareACZ();
@@ -113,10 +148,10 @@ namespace Robust.Server.ServerStatus
 
             // Automatic - pass to ACZ
             // Unfortunately, we still can't divine engine version.
-            var engineVersion = _configurationManager.GetCVar(CVars.BuildEngineVersion);
+            var engineVersion = _cfg.GetCVar(CVars.BuildEngineVersion);
             // Fork ID is an interesting case, we don't want to cause too many redownloads but we also don't want to pollute disk.
             // Call the fork "custom" if there's no explicit ID given.
-            var fork = _configurationManager.GetCVar(CVars.BuildForkId);
+            var fork = _cfg.GetCVar(CVars.BuildForkId);
             if (string.IsNullOrEmpty(fork))
             {
                 fork = "custom";
@@ -125,10 +160,15 @@ namespace Robust.Server.ServerStatus
             {
                 ["engine_version"] = engineVersion,
                 ["fork_id"] = fork,
-                ["version"] = acz.Value.Hash,
+                ["version"] = acz.ManifestHash,
                 // Don't supply a download URL - like supplying an empty self-address
                 ["download_url"] = "",
-                ["hash"] = acz.Value.Hash,
+                ["manifest_download_url"] = "",
+                ["manifest_url"] = "",
+                // Pass acz so the launcher knows where to find the downloads.
+                ["acz"] = true,
+                ["hash"] = acz.ZipHash,
+                ["manifest_hash"] = acz.ManifestHash
             };
         }
     }

Robust.Server/ServerStatus/StatusHost.cs

@@ -31,7 +31,7 @@ namespace Robust.Server.ServerStatus
     {
         private const string Sawmill = "statushost";
 
-        [Dependency] private readonly IConfigurationManager _configurationManager = default!;
+        [Dependency] private readonly IConfigurationManager _cfg = default!;
         [Dependency] private readonly IServerNetManager _netManager = default!;
         [Dependency] private readonly IPlayerManager _playerManager = default!;
 
@@ -39,6 +39,7 @@ namespace Robust.Server.ServerStatus
         private HttpListener? _listener;
         private TaskCompletionSource? _stopSource;
         private ISawmill _httpSawmill = default!;
+        private ISawmill _aczSawmill = default!;
 
         private string? _serverNameCache;
 
@@ -95,13 +96,14 @@ namespace Robust.Server.ServerStatus
         public void Start()
         {
             _httpSawmill = Logger.GetSawmill($"{Sawmill}.http");
+            _aczSawmill = Logger.GetSawmill($"{Sawmill}.acz");
             RegisterCVars();
 
             // Cache this in a field to avoid thread safety shenanigans.
             // Writes/reads of references are atomic in C# so no further synchronization necessary.
-            _configurationManager.OnValueChanged(CVars.GameHostName, n => _serverNameCache = n, true);
+            _cfg.OnValueChanged(CVars.GameHostName, n => _serverNameCache = n, true);
 
-            if (!_configurationManager.GetCVar(CVars.StatusEnabled))
+            if (!_cfg.GetCVar(CVars.StatusEnabled))
             {
                 return;
             }
@@ -110,7 +112,7 @@ namespace Robust.Server.ServerStatus
 
             _stopSource = new TaskCompletionSource();
             _listener = new HttpListener();
-            _listener.Prefixes.Add($"http://{_configurationManager.GetCVar(CVars.StatusBind)}/");
+            _listener.Prefixes.Add($"http://{_cfg.GetCVar(CVars.StatusBind)}/");
             _listener.Start();
 
             Task.Run(ListenerThread);
@@ -119,7 +121,7 @@ namespace Robust.Server.ServerStatus
         // Not a real thread but whatever.
         private async Task ListenerThread()
         {
-            var maxConnections = _configurationManager.GetCVar(CVars.StatusMaxConnections);
+            var maxConnections = _cfg.GetCVar(CVars.StatusMaxConnections);
             var connectionsSemaphore = new SemaphoreSlim(maxConnections, maxConnections);
             while (true)
             {
@@ -157,6 +159,8 @@ namespace Robust.Server.ServerStatus
 
         private void RegisterCVars()
         {
+            InitAcz();
+
             // Set status host binding to match network manager by default
             SetCVarIfUnmodified(CVars.StatusBind, $"*:{_netManager.Port}");
 
@@ -173,19 +177,22 @@ namespace Robust.Server.ServerStatus
                 SetCVarIfUnmodified(CVars.BuildVersion, info.Version);
                 SetCVarIfUnmodified(CVars.BuildDownloadUrl, info.Download ?? "");
                 SetCVarIfUnmodified(CVars.BuildHash, info.Hash ?? "");
+                SetCVarIfUnmodified(CVars.BuildManifestHash, info.ManifestHash ?? "");
+                SetCVarIfUnmodified(CVars.BuildManifestDownloadUrl, info.ManifestDownloadUrl ?? "");
+                SetCVarIfUnmodified(CVars.BuildManifestUrl, info.ManifestUrl ?? "");
             }
 
             // Automatically determine engine version if no other source has provided a result
             var asmVer = typeof(StatusHost).Assembly.GetName().Version;
             if (asmVer != null)
             {
-                SetCVarIfUnmodified(CVars.BuildEngineVersion, asmVer.ToString(3));
+                SetCVarIfUnmodified(CVars.BuildEngineVersion, asmVer.ToString(4));
             }
 
             void SetCVarIfUnmodified(CVarDef<string> cvar, string val)
             {
-                if (_configurationManager.GetCVar(cvar) == "")
-                    _configurationManager.SetCVar(cvar, val);
+                if (_cfg.GetCVar(cvar) == "")
+                    _cfg.SetCVar(cvar, val);
             }
 
         }
@@ -211,17 +218,33 @@ namespace Robust.Server.ServerStatus
             [property: JsonPropertyName("fork_id")]
             string ForkId,
             [property: JsonPropertyName("version")]
-            string Version);
+            string Version,
+            [property: JsonPropertyName("manifest_hash")]
+            string? ManifestHash,
+            [property: JsonPropertyName("manifest_url")]
+            string? ManifestUrl,
+            [property: JsonPropertyName("manifest_download_url")]
+            string? ManifestDownloadUrl);
 
         private sealed class ContextImpl : IStatusHandlerContext
         {
             private readonly HttpListenerContext _context;
+            private readonly Dictionary<string, string> _responseHeaders;
             public HttpMethod RequestMethod { get; }
             public IPEndPoint RemoteEndPoint => _context.Request.RemoteEndPoint!;
+            public Stream RequestBody => _context.Request.InputStream;
             public Uri Url => _context.Request.Url!;
             public bool IsGetLike => RequestMethod == HttpMethod.Head || RequestMethod == HttpMethod.Get;
             public IReadOnlyDictionary<string, StringValues> RequestHeaders { get; }
 
+            public bool KeepAlive
+            {
+                get => _context.Response.KeepAlive;
+                set => _context.Response.KeepAlive = value;
+            }
+
+            public IDictionary<string, string> ResponseHeaders => _responseHeaders;
+
             public ContextImpl(HttpListenerContext context)
             {
                 _context = context;
@@ -237,16 +260,17 @@ namespace Robust.Server.ServerStatus
                 }
 
                 RequestHeaders = headers;
+                _responseHeaders = new Dictionary<string, string>();
             }
 
             public T? RequestBodyJson<T>()
             {
-                return JsonSerializer.Deserialize<T>(_context.Request.InputStream);
+                return JsonSerializer.Deserialize<T>(RequestBody);
             }
 
             public async Task<T?> RequestBodyJsonAsync<T>()
             {
-                return await JsonSerializer.DeserializeAsync<T>(_context.Request.InputStream);
+                return await JsonSerializer.DeserializeAsync<T>(RequestBody);
             }
 
             public void Respond(string text, HttpStatusCode code = HttpStatusCode.OK, string contentType = MediaTypeNames.Text.Plain)
@@ -290,6 +314,16 @@ namespace Robust.Server.ServerStatus
                 _context.Response.Close();
             }
 
+            public Task RespondNoContentAsync()
+            {
+                RespondShared();
+
+                _context.Response.StatusCode = (int) HttpStatusCode.NoContent;
+                _context.Response.Close();
+
+                return Task.CompletedTask;
+            }
+
             public Task RespondAsync(string text, HttpStatusCode code = HttpStatusCode.OK, string contentType = "text/plain")
             {
                 return RespondAsync(text, (int) code, contentType);
@@ -297,6 +331,8 @@ namespace Robust.Server.ServerStatus
 
             public async Task RespondAsync(string text, int code = 200, string contentType = "text/plain")
             {
+                RespondShared();
+
                 _context.Response.StatusCode = code;
                 _context.Response.ContentType = contentType;
 
@@ -315,6 +351,8 @@ namespace Robust.Server.ServerStatus
 
             public async Task RespondAsync(byte[] data, int code = 200, string contentType = "text/plain")
             {
+                RespondShared();
+
                 _context.Response.StatusCode = code;
                 _context.Response.ContentType = contentType;
                 _context.Response.ContentLength64 = data.Length;
@@ -341,6 +379,8 @@ namespace Robust.Server.ServerStatus
 
             public void RespondJson(object jsonData, HttpStatusCode code = HttpStatusCode.OK)
             {
+                RespondShared();
+
                 _context.Response.ContentType = "application/json";
 
                 JsonSerializer.Serialize(_context.Response.OutputStream, jsonData);
@@ -350,12 +390,31 @@ namespace Robust.Server.ServerStatus
 
             public async Task RespondJsonAsync(object jsonData, HttpStatusCode code = HttpStatusCode.OK)
             {
+                RespondShared();
+
                 _context.Response.ContentType = "application/json";
 
                 await JsonSerializer.SerializeAsync(_context.Response.OutputStream, jsonData);
 
                 _context.Response.Close();
             }
+
+            public Task<Stream> RespondStreamAsync(HttpStatusCode code = HttpStatusCode.OK)
+            {
+                RespondShared();
+
+                _context.Response.StatusCode = (int) code;
+
+                return Task.FromResult(_context.Response.OutputStream);
+            }
+
+            private void RespondShared()
+            {
+                foreach (var (header, value) in _responseHeaders)
+                {
+                    _context.Response.AddHeader(header, value);
+                }
+            }
         }
     }
 }

Robust.Shared/CVars.cs

@@ -426,12 +426,30 @@ namespace Robust.Shared
         public static readonly CVarDef<string> BuildDownloadUrl =
             CVarDef.Create("build.download_url", string.Empty, CVar.SERVERONLY);
 
+        /// <summary>
+        /// URL of the content manifest the launcher should download to connect to this server.
+        /// </summary>
+        public static readonly CVarDef<string> BuildManifestUrl =
+            CVarDef.Create("build.manifest_url", string.Empty, CVar.SERVERONLY);
+
+        /// <summary>
+        /// URL at which the launcher can download the manifest game files.
+        /// </summary>
+        public static readonly CVarDef<string> BuildManifestDownloadUrl =
+            CVarDef.Create("build.manifest_download_url", string.Empty, CVar.SERVERONLY);
+
         /// <summary>
         /// SHA-256 hash of the content pack hosted at <c>build.download_url</c>
         /// </summary>
         public static readonly CVarDef<string> BuildHash =
             CVarDef.Create("build.hash", "", CVar.SERVERONLY);
 
+        /// <summary>
+        /// SHA-256 hash of the manifest hosted at <c>build.manifest_url</c>
+        /// </summary>
+        public static readonly CVarDef<string> BuildManifestHash =
+            CVarDef.Create("build.manifest_hash", "", CVar.SERVERONLY);
+
         /*
          * WATCHDOG
          */
@@ -1075,5 +1093,57 @@ namespace Robust.Shared
         /// </summary>
         public static readonly CVarDef<int> HubAdvertiseInterval =
             CVarDef.Create("hub.advertise_interval", 120, CVar.SERVERONLY);
+
+        /*
+         * ACZ
+         */
+
+        /// <summary>
+        /// Whether to use stream compression instead of per-file compression when transmitting ACZ data.
+        /// Enabling stream compression significantly reduces bandwidth usage of downloads,
+        /// but increases server and launcher CPU load. It also makes final files stored on the client compressed less.
+        /// </summary>
+        public static readonly CVarDef<bool> AczStreamCompress =
+            CVarDef.Create("acz.stream_compress", false, CVar.SERVERONLY);
+
+        /// <summary>
+        /// ZSTD Compression level to use when doing ACZ stream compressed.
+        /// </summary>
+        public static readonly CVarDef<int> AczStreamCompressLevel =
+            CVarDef.Create("acz.stream_compress_level", 3, CVar.SERVERONLY);
+
+        /// <summary>
+        /// Whether to do compression on individual files for ACZ downloads.
+        /// Automatically forced off if stream compression is enabled.
+        /// </summary>
+        public static readonly CVarDef<bool> AczBlobCompress =
+            CVarDef.Create("acz.blob_compress", true, CVar.SERVERONLY);
+
+        /// <summary>
+        /// ZSTD Compression level to use for individual file compression.
+        /// </summary>
+        public static readonly CVarDef<int> AczBlobCompressLevel =
+            CVarDef.Create("acz.blob_compress_level", 14, CVar.SERVERONLY);
+
+        // Could consider using a ratio for this?
+        /// <summary>
+        /// Amount of bytes that need to be saved by compression for the compression to be "worth it".
+        /// </summary>
+        public static readonly CVarDef<int> AczBlobCompressSaveThreshold =
+            CVarDef.Create("acz.blob_compress_save_threshold", 14, CVar.SERVERONLY);
+
+        /// <summary>
+        /// Whether to ZSTD compress the ACZ manifest.
+        /// If this is enabled (the default) then non-compressed manifest requests will be decompressed live.
+        /// </summary>
+        public static readonly CVarDef<bool> AczManifestCompress =
+            CVarDef.Create("acz.manifest_compress", true, CVar.SERVERONLY);
+
+        /// <summary>
+        /// Compression level for ACZ manifest compression.
+        /// </summary>
+        public static readonly CVarDef<int> AczManifestCompressLevel =
+            CVarDef.Create("acz.manifest_compress_level", 14, CVar.SERVERONLY);
+
     }
 }

Robust.Shared/Configuration/ConfigurationManager.cs

@@ -4,9 +4,11 @@ using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
 using Nett;
 using Robust.Shared.Log;
 using Robust.Shared.Utility;
+using Robust.Shared.Utility.Collections;
 
 namespace Robust.Shared.Configuration
 {
@@ -21,6 +23,8 @@ namespace Robust.Shared.Configuration
         private string? _configFile;
         protected bool _isServer;
 
+        protected readonly ReaderWriterLockSlim Lock = new();
+
         /// <summary>
         ///     Constructs a new ConfigurationManager.
         /// </summary>
@@ -35,6 +39,8 @@ namespace Robust.Shared.Configuration
 
         public virtual void Shutdown()
         {
+            using var _ = Lock.WriteGuard();
+
             _configVars.Clear();
             _configFile = null;
         }
@@ -46,7 +52,18 @@ namespace Robust.Shared.Configuration
             {
                 var tblRoot = Toml.ReadFile(configFile);
 
-                ProcessTomlObject(tblRoot);
+                var callbackEvents = new ValueList<ValueChangedInvoke>();
+
+                // Ensure callbacks are raised OUTSIDE the write lock.
+                using (Lock.WriteGuard())
+                {
+                    ProcessTomlObject(tblRoot, ref callbackEvents);
+                }
+
+                foreach (var callback in callbackEvents)
+                {
+                    InvokeValueChanged(callback);
+                }
 
                 _configFile = configFile;
                 Logger.InfoS("cfg", $"Configuration Loaded from '{Path.GetFullPath(configFile)}'");
@@ -66,8 +83,12 @@ namespace Robust.Shared.Configuration
         /// A recursive function that walks over the config tree, transforming all key nodes into CVars.
         /// </summary>
         /// <param name="obj">The root table of the TOML document.</param>
+        /// <param name="changedInvokes">List of CVars that will need to have their InvokeValueChanged ran.</param>
         /// <param name="tablePath">For internal use only, the current path to the node.</param>
-        private void ProcessTomlObject(TomlObject obj, string tablePath = "")
+        private void ProcessTomlObject(
+            TomlObject obj,
+            ref ValueList<ValueChangedInvoke> changedInvokes,
+            string tablePath = "")
         {
             if (obj is TomlTable table) // this is a table
             {
@@ -80,7 +101,7 @@ namespace Robust.Shared.Configuration
                     else
                         newPath = tablePath + kvTml.Key;
 
-                    ProcessTomlObject(kvTml.Value, newPath);
+                    ProcessTomlObject(kvTml.Value, ref changedInvokes, newPath);
                 }
             }
             else // this is a key, add CVar
@@ -91,7 +112,8 @@ namespace Robust.Shared.Configuration
                 {
                     // overwrite the value with the saved one
                     cfgVar.Value = tomlValue;
-                    InvokeValueChanged(cfgVar, cfgVar.Value);
+                    if (SetupInvokeValueChanged(cfgVar, tomlValue) is { } invoke)
+                        changedInvokes.Add(invoke);
                 }
                 else
                 {
@@ -118,73 +140,76 @@ namespace Robust.Shared.Configuration
             {
                 var tblRoot = Toml.Create();
 
-                foreach (var (name, cVar) in _configVars)
+                using (Lock.ReadGuard())
                 {
-                    var value = cVar.Value;
-                    if (value == null && cVar.Registered)
+                    foreach (var (name, cVar) in _configVars)
                     {
-                        value = cVar.DefaultValue;
-                    }
-
-                    if (value == null)
-                    {
-                        Logger.ErrorS("cfg",
-                            $"CVar {name} has no value or default value, was the default value registered as null?");
-                        continue;
-                    }
-
-                    // Don't write if Archive flag is not set.
-                    // Don't write if the cVar is the default value.
-                    if (!cVar.ConfigModified &&
-                        (cVar.Flags & CVar.ARCHIVE) == 0 || value.Equals(cVar.DefaultValue))
-                    {
-                        continue;
-                    }
-
-                    var keyIndex = name.LastIndexOf(TABLE_DELIMITER);
-                    var tblPath = name.Substring(0, keyIndex).Split(TABLE_DELIMITER);
-                    var keyName = name.Substring(keyIndex + 1);
-
-                    // locate the Table in the config tree
-                    var table = tblRoot;
-                    foreach (var curTblName in tblPath)
-                    {
-                        if (!table.TryGetValue(curTblName, out TomlObject tblObject))
+                        var value = cVar.Value;
+                        if (value == null && cVar.Registered)
                         {
-                            tblObject = table.Add(curTblName, new Dictionary<string, TomlObject>()).Added;
+                            value = cVar.DefaultValue;
                         }
 
-                        table = tblObject as TomlTable ?? throw new InvalidConfigurationException(
-                            $"[CFG] Object {curTblName} is being used like a table, but it is a {tblObject}. Are your CVar names formed properly?");
-                    }
+                        if (value == null)
+                        {
+                            Logger.ErrorS("cfg",
+                                $"CVar {name} has no value or default value, was the default value registered as null?");
+                            continue;
+                        }
 
-                    //runtime unboxing, either this or generic hell... ¯\_(ツ)_/¯
-                    switch (value)
-                    {
-                        case Enum val:
-                            table.Add(keyName, (int) (object) val); // asserts Enum value != (ulong || long)
-                            break;
-                        case int val:
-                            table.Add(keyName, val);
-                            break;
-                        case long val:
-                            table.Add(keyName, val);
-                            break;
-                        case bool val:
-                            table.Add(keyName, val);
-                            break;
-                        case string val:
-                            table.Add(keyName, val);
-                            break;
-                        case float val:
-                            table.Add(keyName, val);
-                            break;
-                        case double val:
-                            table.Add(keyName, val);
-                            break;
-                        default:
-                            Logger.WarningS("cfg", $"Cannot serialize '{name}', unsupported type.");
-                            break;
+                        // Don't write if Archive flag is not set.
+                        // Don't write if the cVar is the default value.
+                        if (!cVar.ConfigModified &&
+                            (cVar.Flags & CVar.ARCHIVE) == 0 || value.Equals(cVar.DefaultValue))
+                        {
+                            continue;
+                        }
+
+                        var keyIndex = name.LastIndexOf(TABLE_DELIMITER);
+                        var tblPath = name.Substring(0, keyIndex).Split(TABLE_DELIMITER);
+                        var keyName = name.Substring(keyIndex + 1);
+
+                        // locate the Table in the config tree
+                        var table = tblRoot;
+                        foreach (var curTblName in tblPath)
+                        {
+                            if (!table.TryGetValue(curTblName, out TomlObject tblObject))
+                            {
+                                tblObject = table.Add(curTblName, new Dictionary<string, TomlObject>()).Added;
+                            }
+
+                            table = tblObject as TomlTable ?? throw new InvalidConfigurationException(
+                                $"[CFG] Object {curTblName} is being used like a table, but it is a {tblObject}. Are your CVar names formed properly?");
+                        }
+
+                        //runtime unboxing, either this or generic hell... ¯\_(ツ)_/¯
+                        switch (value)
+                        {
+                            case Enum val:
+                                table.Add(keyName, (int)(object)val); // asserts Enum value != (ulong || long)
+                                break;
+                            case int val:
+                                table.Add(keyName, val);
+                                break;
+                            case long val:
+                                table.Add(keyName, val);
+                                break;
+                            case bool val:
+                                table.Add(keyName, val);
+                                break;
+                            case string val:
+                                table.Add(keyName, val);
+                                break;
+                            case float val:
+                                table.Add(keyName, val);
+                                break;
+                            case double val:
+                                table.Add(keyName, val);
+                                break;
+                            default:
+                                Logger.WarningS("cfg", $"Cannot serialize '{name}', unsupported type.");
+                                break;
+                        }
                     }
                 }
 
@@ -220,6 +245,8 @@ namespace Robust.Shared.Configuration
                 return;
             }
 
+            using var _ = Lock.WriteGuard();
+
             if (_configVars.TryGetValue(name, out var cVar))
             {
                 if (cVar.Registered)
@@ -253,12 +280,16 @@ namespace Robust.Shared.Configuration
         public void OnValueChanged<T>(string name, Action<T> onValueChanged, bool invokeImmediately = false)
             where T : notnull
         {
-            var reg = _configVars[name];
-            var exDel = (Action<T>?) reg.ValueChanged;
-            exDel += onValueChanged;
-            reg.ValueChanged = exDel;
 
-            reg.ValueChangedInvoker ??= (del, v) => ((Action<T>) del)((T) v);
+            using (Lock.WriteGuard())
+            {
+                var reg = _configVars[name];
+                var exDel = (Action<T>?) reg.ValueChanged;
+                exDel += onValueChanged;
+                reg.ValueChanged = exDel;
+
+                reg.ValueChangedInvoker ??= (del, v) => ((Action<T>) del)((T) v);
+            }
 
             if (invokeImmediately)
             {
@@ -273,6 +304,8 @@ namespace Robust.Shared.Configuration
 
         public void UnsubValueChanged<T>(string name, Action<T> onValueChanged) where T : notnull
         {
+            using var _ = Lock.WriteGuard();
+
             var reg = _configVars[name];
             var exDel = (Action<T>?) reg.ValueChanged;
             exDel -= onValueChanged;
@@ -315,13 +348,17 @@ namespace Robust.Shared.Configuration
         /// <inheritdoc />
         public bool IsCVarRegistered(string name)
         {
+            using var _ = Lock.ReadGuard();
             return _configVars.TryGetValue(name, out var cVar) && cVar.Registered;
         }
 
         /// <inheritdoc />
         public IEnumerable<string> GetRegisteredCVars()
         {
-            return _configVars.Select(p => p.Key);
+            using var _ = Lock.ReadGuard();
+            // Have to .ToArray() so the lock is held for the whole iteration operation.
+            // This function is only currently used for the cvar ? command so I'm not too worried.
+            return _configVars.Select(p => p.Key).ToArray();
         }
 
         /// <inheritdoc />
@@ -332,21 +369,29 @@ namespace Robust.Shared.Configuration
 
         private void SetCVarInternal(string name, object value)
         {
-            //TODO: Make flags work, required non-derpy net system.
-            if (_configVars.TryGetValue(name, out var cVar) && cVar.Registered)
-            {
-                if (!Equals(cVar.OverrideValueParsed ?? cVar.Value, value))
-                {
-                    // Setting an overriden var just turns off the override, basically.
-                    cVar.OverrideValue = null;
-                    cVar.OverrideValueParsed = null;
+            ValueChangedInvoke? invoke = null;
 
-                    cVar.Value = value;
-                    InvokeValueChanged(cVar, value);
+            using (Lock.WriteGuard())
+            {
+                //TODO: Make flags work, required non-derpy net system.
+                if (_configVars.TryGetValue(name, out var cVar) && cVar.Registered)
+                {
+                    if (!Equals(cVar.OverrideValueParsed ?? cVar.Value, value))
+                    {
+                        // Setting an overriden var just turns off the override, basically.
+                        cVar.OverrideValue = null;
+                        cVar.OverrideValueParsed = null;
+
+                        cVar.Value = value;
+                        invoke = SetupInvokeValueChanged(cVar, value);
+                    }
                 }
+                else
+                    throw new InvalidConfigurationException($"Trying to set unregistered variable '{name}'");
             }
-            else
-                throw new InvalidConfigurationException($"Trying to set unregistered variable '{name}'");
+
+            if (invoke != null)
+                InvokeValueChanged(invoke.Value);
         }
 
         public void SetCVar<T>(CVarDef<T> def, T value) where T : notnull
@@ -357,6 +402,7 @@ namespace Robust.Shared.Configuration
         /// <inheritdoc />
         public T GetCVar<T>(string name)
         {
+            using var _ = Lock.ReadGuard();
             if (_configVars.TryGetValue(name, out var cVar) && cVar.Registered)
                 //TODO: Make flags work, required non-derpy net system.
                 return (T) (GetConfigVarValue(cVar))!;
@@ -371,6 +417,7 @@ namespace Robust.Shared.Configuration
 
         public Type GetCVarType(string name)
         {
+            using var _ = Lock.ReadGuard();
             if (!_configVars.TryGetValue(name, out var cVar) || !cVar.Registered)
             {
                 throw new InvalidConfigurationException($"Trying to get type of unregistered variable '{name}'");
@@ -387,24 +434,35 @@ namespace Robust.Shared.Configuration
 
         public void OverrideConVars(IEnumerable<(string key, string value)> cVars)
         {
-            foreach (var (key, value) in cVars)
+            var invokes = new ValueList<ValueChangedInvoke>();
+
+            using (Lock.WriteGuard())
             {
-                if (_configVars.TryGetValue(key, out var cfgVar))
+                foreach (var (key, value) in cVars)
                 {
-                    cfgVar.OverrideValue = value;
-                    if (cfgVar.Registered)
+                    if (_configVars.TryGetValue(key, out var cfgVar))
                     {
-                        cfgVar.OverrideValueParsed = ParseOverrideValue(value, cfgVar.DefaultValue?.GetType());
-                        InvokeValueChanged(cfgVar, cfgVar.OverrideValueParsed);
+                        cfgVar.OverrideValue = value;
+                        if (cfgVar.Registered)
+                        {
+                            cfgVar.OverrideValueParsed = ParseOverrideValue(value, cfgVar.DefaultValue?.GetType());
+                            if (SetupInvokeValueChanged(cfgVar, cfgVar.OverrideValueParsed) is { } invoke)
+                                invokes.Add(invoke);
+                        }
+                    }
+                    else
+                    {
+                        //or add another unregistered CVar
+                        //Note: the defaultValue is arbitrarily 0, it will get overwritten when the cvar is registered.
+                        var cVar = new ConfigVar(key, 0, CVar.NONE) {OverrideValue = value};
+                        _configVars.Add(key, cVar);
                     }
                 }
-                else
-                {
-                    //or add another unregistered CVar
-                    //Note: the defaultValue is arbitrarily 0, it will get overwritten when the cvar is registered.
-                    var cVar = new ConfigVar(key, 0, CVar.NONE) {OverrideValue = value};
-                    _configVars.Add(key, cVar);
-                }
+            }
+
+            foreach (var invoke in invokes)
+            {
+                InvokeValueChanged(invoke);
             }
         }
 
@@ -461,9 +519,17 @@ namespace Robust.Shared.Configuration
             }
         }
 
-        private static void InvokeValueChanged(ConfigVar var, object value)
+        private static void InvokeValueChanged(ValueChangedInvoke invoke)
         {
-            var.ValueChangedInvoker?.Invoke(var.ValueChanged!, value);
+            invoke.Invoker.Invoke(invoke.ValueChanged, invoke.Value);
+        }
+
+        private static ValueChangedInvoke? SetupInvokeValueChanged(ConfigVar var, object value)
+        {
+            if (var.ValueChangedInvoker == null)
+                return null;
+
+            return new ValueChangedInvoke(var.ValueChangedInvoker, var.ValueChanged!, value);
         }
 
         /// <summary>
@@ -530,6 +596,14 @@ namespace Robust.Shared.Configuration
             public string? OverrideValue { get; set; }
             public object? OverrideValueParsed { get; set; }
         }
+
+        /// <summary>
+        /// All data we need to invoke a deferred ValueChanged handler outside of a write lock.
+        /// </summary>
+        private record struct ValueChangedInvoke(
+            Action<Delegate, object> Invoker,
+            Delegate ValueChanged,
+            object Value);
     }
 
     [Serializable]

Robust.Shared/Configuration/IConfigurationManager.cs

@@ -6,6 +6,13 @@ namespace Robust.Shared.Configuration
     /// <summary>
     /// Stores and manages global configuration variables.
     /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Accessing (getting/setting) main CVars is thread safe.
+    /// Note that value-changed callbacks are ran synchronously from the thread using <see cref="SetCVar"/>,
+    /// so it is not recommended to modify CVars from other threads.
+    /// </para>
+    /// </remarks>
     public interface IConfigurationManager
     {
         /// <summary>

Robust.Shared/Configuration/NetConfigurationManager.cs

@@ -190,6 +190,8 @@ namespace Robust.Shared.Configuration
                 return;
             }
 
+            using var _ = Lock.ReadGuard();
+
             foreach (var (name, value) in networkedVars)
             {
                 if (!_configVars.TryGetValue(name, out var cVar))
@@ -219,6 +221,8 @@ namespace Robust.Shared.Configuration
         /// <inheritdoc />
         public T GetClientCVar<T>(INetChannel channel, string name)
         {
+            using var _ = Lock.ReadGuard();
+
             if (!_configVars.TryGetValue(name, out var cVar) || !cVar.Registered)
                 throw new InvalidConfigurationException($"Trying to get unregistered variable '{name}'");
 
@@ -233,42 +237,45 @@ namespace Robust.Shared.Configuration
         /// <inheritdoc />
         public override void SetCVar(string name, object value)
         {
-            if (_configVars.TryGetValue(name, out var cVar) && cVar.Registered)
+            CVar flags;
+            using (Lock.ReadGuard())
             {
-                if (_netManager.IsClient)
+                if (_configVars.TryGetValue(name, out var cVar) && cVar.Registered)
                 {
-                    if (_netManager.IsConnected)
+                    flags = cVar.Flags;
+                    if (_netManager.IsClient)
                     {
-                        if ((cVar.Flags & CVar.NOT_CONNECTED) != 0)
+                        if (_netManager.IsConnected)
                         {
-                            Logger.WarningS("cfg", $"'{name}' can only be changed when not connected to a server.");
+                            if ((cVar.Flags & CVar.NOT_CONNECTED) != 0)
+                            {
+                                Logger.WarningS("cfg", $"'{name}' can only be changed when not connected to a server.");
+                                return;
+                            }
+                        }
+
+                        if ((cVar.Flags & CVar.SERVER) != 0)
+                        {
+                            Logger.WarningS("cfg", $"Only the server can change '{name}'.");
                             return;
                         }
                     }
-
-                    if ((cVar.Flags & CVar.SERVER) != 0)
-                    {
-                        Logger.WarningS("cfg", $"Only the server can change '{name}'.");
-                        return;
-                    }
                 }
-            }
-            else
-            {
-                throw new InvalidConfigurationException($"Trying to set unregistered variable '{name}'");
+                else
+                {
+                    throw new InvalidConfigurationException($"Trying to set unregistered variable '{name}'");
+                }
             }
 
             // Actually set the CVar
             base.SetCVar(name, value);
 
-            var cvar = _configVars[name];
+            if ((flags & CVar.REPLICATED) == 0)
+                return;
 
             // replicate if needed
             if (_netManager.IsClient)
             {
-                if ((cvar.Flags & CVar.REPLICATED) == 0)
-                    return;
-
                 var msg = _netManager.CreateNetMessage<MsgConVars>();
                 msg.Tick = _timing.CurTick;
                 msg.NetworkedVars = new List<(string name, object value)>
@@ -279,9 +286,6 @@ namespace Robust.Shared.Configuration
             }
             else // Server
             {
-                if ((cvar.Flags & CVar.REPLICATED) == 0)
-                    return;
-
                 var msg = _netManager.CreateNetMessage<MsgConVars>();
                 msg.Tick = _timing.CurTick;
                 msg.NetworkedVars = new List<(string name, object value)>
@@ -327,6 +331,8 @@ namespace Robust.Shared.Configuration
 
         private List<(string name, object value)> GetReplicatedVars()
         {
+            using var _ = Lock.ReadGuard();
+
             var nwVars = new List<(string name, object value)>();
 
             foreach (var cVar in _configVars.Values)

Robust.Shared/Robust.Shared.csproj

@@ -19,7 +19,8 @@
     <PackageReference Include="YamlDotNet" Version="9.1.4" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="5.0.0" />
     <PackageReference Include="Linguini.Bundle" Version="0.1.3" />
-    <PackageReference Include="SpaceWizards.Sodium" Version="0.1.0" />
+    <PackageReference Include="SharpZstd.Interop" Version="1.5.2-beta1" />
+    <PackageReference Include="SpaceWizards.Sodium" Version="0.2.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Lidgren.Network\Lidgren.Network.csproj" />

Robust.Shared/Utility/BufferHelpers.cs

@@ -0,0 +1,41 @@
+using System.Buffers;
+using System.Numerics;
+
+namespace Robust.Shared.Utility;
+
+/// <summary>
+/// Helpers for dealing with buffer-like arrays.
+/// </summary>
+public static class BufferHelpers
+{
+    /// <summary>
+    /// Resize the given buffer to the next power of two that fits the needed size.
+    /// The contents of the buffer are NOT preserved if resized.
+    /// </summary>
+    public static void EnsureBuffer<T>(ref T[] buf, int minimumLength)
+    {
+        if (buf.Length >= minimumLength)
+            return;
+
+        buf = new T[FittingPowerOfTwo(minimumLength)];
+    }
+
+    /// <summary>
+    /// Resize the given buffer to the next power of two that fits the needed size.
+    /// Takes an array pool to rent/return with.
+    /// The contents of the buffer are NOT preserved across resizes.
+    /// </summary>
+    public static void EnsurePooledBuffer<T>(ref T[] buf, ArrayPool<T> pool, int minimumLength)
+    {
+        if (buf.Length >= minimumLength)
+            return;
+
+        pool.Return(buf);
+        buf = pool.Rent(minimumLength);
+    }
+
+    /// <summary>
+    /// Calculate the smallest power of two that fits the required size.
+    /// </summary>
+    public static int FittingPowerOfTwo(int size) => 2 << BitOperations.Log2((uint)size - 1);
+}

Robust.Shared/Utility/Collections/ValueList.cs

@@ -0,0 +1,487 @@
+// This file includes code based on the List<T> class from https://github.com/dotnet/runtime/
+// The original code is Copyright © .NET Foundation and Contributors. All rights reserved. Licensed under the MIT License (MIT).
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.CompilerServices;
+
+namespace Robust.Shared.Utility.Collections;
+
+/// <summary>
+/// Implementation of <see cref="List{T}"/> that is stored in a struct instead.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Storing this implementation in a struct reduces GC and memory overhead from list instances drastically.
+/// It is only recommended you use this class for private data;
+/// public APIs probably shouldn't expose it unless you know what you're doing.
+/// </para>
+/// <para>
+/// This implementation does not complain if you modify it during iteration. Be careful!
+/// </para>
+/// <para>
+/// The implementation uses an array to store the contained items.
+/// This array may be larger (<see cref="Capacity"/>) than the amount of "actual" items stored (<see cref="Count"/>).
+/// Adding or removing elements to the list shrinks or grows the available capacity at the end of the array.
+/// If there is no remaining capacity left when inserting,
+/// a new, larger, array is allocated and elements are copied over.
+/// </para>
+/// </remarks>
+/// <typeparam name="T">The type of item to store in the list.</typeparam>
+public struct ValueList<T> : IEnumerable<T>
+{
+    private const int DefaultCapacity = 4;
+
+    // List can be null so that the list instance is valid if = defaulted.
+    // Null backing list is equal to empty array everywhere.
+    // It follows from this that having a count or capacity > 0 means the list is not null.
+    private T[]? _items;
+
+    // Constructs a List with a given initial capacity. The list is
+    // initially empty, but will have room for the given number of elements
+    // before any reallocations are required.
+    //
+    public ValueList(int capacity)
+    {
+        _items = capacity == 0 ? null : new T[capacity];
+        Count = 0;
+    }
+
+    /// <summary>
+    /// Create a list by copying the contents from another enumerable.
+    /// </summary>
+    /// <param name="collection">The enumerable to copy the items from.</param>
+    public ValueList(IEnumerable<T> collection)
+    {
+        _items = collection.ToArray();
+        Count = _items.Length;
+    }
+
+    /// <summary>
+    /// Create a list by taking ownership of an existing array.
+    /// Mutations of the list may mutate the passed array.
+    /// The count and capacity of the list are both set equal to the array length.
+    /// </summary>
+    /// <remarks>
+    /// If null is passed, it is treated equivalently to an empty array.
+    /// </remarks>
+    public static ValueList<T> OwningArray(T[]? array)
+    {
+        ValueList<T> list = default;
+        list._items = array;
+        list.Count = list.Capacity;
+        return list;
+    }
+
+    /// <summary>
+    /// Create a list by taking ownership of an existing array.
+    /// Mutations of the list may mutate the passed array.
+    /// The capacity is set to the length of the list.
+    /// The count can be set separately if the array has more space than valid items.
+    /// </summary>
+    /// <remarks>
+    /// If null is passed, it is treated equivalently to an empty array.
+    /// </remarks>
+    /// <exception cref="ArgumentException">
+    /// Thrown if count is negative or if count is greater than the array capacity.
+    /// </exception>
+    public static ValueList<T> OwningArray(T[]? array, int count)
+    {
+        ValueList<T> list = default;
+        list._items = array;
+
+        if (count < 0)
+            throw new ArgumentException("Count cannot be negative.");
+
+        if (count >= list.Capacity)
+            throw new ArgumentException("Count cannot be greater than the size of the array.");
+
+        list.Count = count;
+        return list;
+    }
+
+    public int Count { get; private set; }
+
+    // Sets or Gets the element at the given index.
+    public readonly ref T this[int index]
+    {
+        get
+        {
+            // Following trick can reduce the range check by one
+            if ((uint)index >= (uint)Count)
+                throw new IndexOutOfRangeException();
+
+            return ref _items![index];
+        }
+    }
+
+    public int Capacity
+    {
+        readonly get => _items?.Length ?? 0;
+        set
+        {
+            if (value < Count)
+                throw new ArgumentException("Cannot set capacity lower than contained count");
+
+            if (value == Capacity)
+                return;
+
+            if (value > 0)
+            {
+                var newItems = new T[value];
+                if (Count > 0)
+                    Array.Copy(_items!, newItems, Count);
+
+                _items = newItems;
+            }
+            else
+            {
+                _items = null;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Span containing the items inside the list.
+    /// Note that resizing of the backing array will cause this span to be invalidated.
+    /// </summary>
+    public readonly Span<T> Span => new(_items, 0, Count);
+
+    // Adds the given object to the end of this list. The size of the list is
+    // increased by one. If required, the capacity of the list is doubled
+    // before adding the new element.
+    //
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public void Add(T item)
+    {
+        var array = _items;
+        var size = Count;
+        if ((uint)size < (uint)Capacity)
+        {
+            Count = size + 1;
+            array![size] = item;
+        }
+        else
+        {
+            AddWithResize(item);
+        }
+    }
+
+    // Non-inline from List.Add to improve its code quality as uncommon path
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private void AddWithResize(T item)
+    {
+        Debug.Assert(Count == Capacity);
+
+        var size = Count;
+        Grow(size + 1);
+        Count = size + 1;
+        _items![size] = item;
+    }
+
+    // Clears the contents of List.
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public void Clear()
+    {
+        if (RuntimeHelpers.IsReferenceOrContainsReferences<T>())
+        {
+            var size = Count;
+            Count = 0;
+            if (size > 0)
+            {
+                Array.Clear(_items!, 0, size); // Clear the elements so that the gc can reclaim the references.
+            }
+        }
+        else
+        {
+            Count = 0;
+        }
+    }
+
+    // Contains returns true if the specified element is in the List.
+    // It does a linear, O(n) search.  Equality is determined by calling
+    // EqualityComparer<T>.Default.Equals().
+    //
+    public readonly bool Contains(T item)
+    {
+        return IndexOf(item) >= 0;
+    }
+
+    /// <summary>
+    /// Ensures that the capacity of this list is at least the specified <paramref name="capacity"/>.
+    /// If the current capacity of the list is less than specified <paramref name="capacity"/>,
+    /// the capacity is increased by continuously twice current capacity until it is at least the specified <paramref name="capacity"/>.
+    /// </summary>
+    /// <param name="capacity">The minimum capacity to ensure.</param>
+    /// <returns>The new capacity of this list.</returns>
+    public int EnsureCapacity(int capacity)
+    {
+        if (capacity < 0)
+            throw new ArgumentException("Capacity cannot be negative");
+
+        if (Capacity < capacity)
+            Grow(capacity);
+
+        return _items!.Length;
+    }
+
+    /// <summary>
+    /// Increase the capacity of this list to at least the specified <paramref name="capacity"/>.
+    /// </summary>
+    /// <param name="capacity">The minimum capacity to ensure.</param>
+    private void Grow(int capacity)
+    {
+        Debug.Assert(Capacity < capacity);
+
+        int newcapacity = Capacity == 0 ? DefaultCapacity : 2 * _items!.Length;
+
+        // Allow the list to grow to maximum possible capacity (~2G elements) before encountering overflow.
+        // Note that this check works even when _items.Length overflowed thanks to the (uint) cast
+        if ((uint)newcapacity > Array.MaxLength) newcapacity = Array.MaxLength;
+
+        // If the computed capacity is still less than specified, set to the original argument.
+        // Capacities exceeding Array.MaxLength will be surfaced as OutOfMemoryException by Array.Resize.
+        if (newcapacity < capacity) newcapacity = capacity;
+
+        Capacity = newcapacity;
+    }
+
+    // Returns an enumerator for this list with the given
+    // permission for removal of elements. If modifications made to the list
+    // while an enumeration is in progress, the MoveNext and
+    // GetObject methods of the enumerator will throw an exception.
+    //
+    public readonly Enumerator GetEnumerator()
+        => new Enumerator(this);
+
+    IEnumerator<T> IEnumerable<T>.GetEnumerator()
+        => new Enumerator(this);
+
+    IEnumerator IEnumerable.GetEnumerator()
+        => new Enumerator(this);
+
+
+    // Returns the index of the first occurrence of a given value in a range of
+    // this list. The list is searched forwards from beginning to end.
+    // The elements of the list are compared to the given value using the
+    // Object.Equals method.
+    //
+    // This method uses the Array.IndexOf method to perform the
+    // search.
+    //
+    public readonly int IndexOf(T item)
+        => _items == null ? -1 : Array.IndexOf(_items, item, 0, Count);
+
+    // Returns the index of the first occurrence of a given value in a range of
+    // this list. The list is searched forwards, starting at index
+    // index and ending at count number of elements. The
+    // elements of the list are compared to the given value using the
+    // Object.Equals method.
+    //
+    // This method uses the Array.IndexOf method to perform the
+    // search.
+    //
+    public readonly int IndexOf(T item, int index)
+    {
+        if (index > Count)
+            throw new ArgumentOutOfRangeException();
+
+        return _items == null ? -1 : Array.IndexOf(_items, item, index, Count - index);
+    }
+
+    // Returns the index of the first occurrence of a given value in a range of
+    // this list. The list is searched forwards, starting at index
+    // index and upto count number of elements. The
+    // elements of the list are compared to the given value using the
+    // Object.Equals method.
+    //
+    // This method uses the Array.IndexOf method to perform the
+    // search.
+    //
+    public readonly int IndexOf(T item, int index, int count)
+    {
+        if (index > Count)
+            throw new ArgumentException("Start index out of bounds");
+
+        if (count < 0 || index > Count - count)
+            throw new ArgumentException("Count out of range");
+
+        return _items == null ? -1 : Array.IndexOf(_items, item, index, count);
+    }
+
+    // Inserts an element into this list at a given index. The size of the list
+    // is increased by one. If required, the capacity of the list is doubled
+    // before inserting the new element.
+    //
+    public void Insert(int index, T item)
+    {
+        // Note that insertions at the end are legal.
+        if ((uint)index > (uint)Count)
+        {
+            throw new ArgumentOutOfRangeException();
+        }
+
+        if (Count == _items!.Length) Grow(Count + 1);
+        if (index < Count)
+        {
+            Array.Copy(_items, index, _items, index + 1, Count - index);
+        }
+
+        _items[index] = item;
+        Count++;
+    }
+
+    // Returns the index of the last occurrence of a given value in a range of
+    // this list. The list is searched backwards, starting at the end
+    // and ending at the first element in the list. The elements of the list
+    // are compared to the given value using the Object.Equals method.
+    //
+    // This method uses the Array.LastIndexOf method to perform the
+    // search.
+    //
+    public readonly int LastIndexOf(T item)
+    {
+        if (Count == 0)
+        {
+            // Special case for empty list
+            return -1;
+        }
+
+        return LastIndexOf(item, Count - 1, Count);
+    }
+
+    // Returns the index of the last occurrence of a given value in a range of
+    // this list. The list is searched backwards, starting at index
+    // index and ending at the first element in the list. The
+    // elements of the list are compared to the given value using the
+    // Object.Equals method.
+    //
+    // This method uses the Array.LastIndexOf method to perform the
+    // search.
+    //
+    public readonly int LastIndexOf(T item, int index)
+    {
+        if (index >= Count)
+            throw new ArgumentOutOfRangeException(nameof(index), "Index out of range");
+
+        return LastIndexOf(item, index, index + 1);
+    }
+
+    // Returns the index of the last occurrence of a given value in a range of
+    // this list. The list is searched backwards, starting at index
+    // index and upto count elements. The elements of
+    // the list are compared to the given value using the Object.Equals
+    // method.
+    //
+    // This method uses the Array.LastIndexOf method to perform the
+    // search.
+    //
+    public readonly int LastIndexOf(T item, int index, int count)
+    {
+        if (Count == 0)
+        {
+            // Special case for empty list
+            return -1;
+        }
+
+        if (index < 0)
+            throw new ArgumentException("Index cannot be negative");
+
+        if (count < 0)
+            throw new ArgumentException("Count cannot be negative");
+
+        if (index >= Count)
+            throw new ArgumentException("Range outside of collection bounds");
+
+        if (count > index + 1)
+            throw new ArgumentException("Range outside of collection bounds");
+
+        return Array.LastIndexOf(_items!, item, index, count);
+    }
+
+    // Removes the element at the given index. The size of the list is
+    // decreased by one.
+    public bool Remove(T item)
+    {
+        var index = IndexOf(item);
+        if (index >= 0)
+        {
+            RemoveAt(index);
+            return true;
+        }
+
+        return false;
+    }
+
+    // Removes the element at the given index. The size of the list is
+    // decreased by one.
+    public void RemoveAt(int index)
+    {
+        if ((uint)index >= (uint)Count)
+            throw new ArgumentOutOfRangeException(nameof(index));
+
+        Count--;
+        if (index < Count)
+            Array.Copy(_items!, index + 1, _items!, index, Count - index);
+
+        if (RuntimeHelpers.IsReferenceOrContainsReferences<T>())
+            _items![Count] = default!;
+    }
+
+    public void Sort() => Span.Sort();
+    public void Sort(IComparer<T>? comparer) => Span.Sort(comparer);
+    public void Sort(Comparison<T> comparison) => Span.Sort(comparison);
+
+    public readonly T[] ToArray() => Span.ToArray();
+
+    // Sets the capacity of this list to the size of the list. This method can
+    // be used to minimize a list's memory overhead once it is known that no
+    // new elements will be added to the list. To completely clear a list and
+    // release all memory referenced by the list, execute the following
+    // statements:
+    //
+    // list.Clear();
+    // list.TrimExcess();
+    //
+    public void TrimExcess()
+    {
+        var threshold = (int)(Capacity * 0.9);
+
+        if (Count < threshold)
+            Capacity = Count;
+    }
+
+    public struct Enumerator : IEnumerator<T>
+    {
+        private readonly ValueList<T> _list;
+        private int _index;
+
+        internal Enumerator(ValueList<T> list)
+        {
+            _index = -1;
+            _list = list;
+        }
+
+        public void Dispose()
+        {
+        }
+
+        public bool MoveNext()
+        {
+            return ++_index < _list.Count;
+        }
+
+        public T Current => RefCurrent;
+        public ref T RefCurrent => ref _list._items![_index];
+
+        object? IEnumerator.Current => Current;
+
+        void IEnumerator.Reset()
+        {
+            _index = -1;
+        }
+    }
+}

Robust.Shared/Utility/LockUtility.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+
+namespace Robust.Shared.Utility;
+
+/// <summary>
+/// Convenience utilities for working with various locking classes.
+/// </summary>
+public static class LockUtility
+{
+    /// <summary>
+    /// Enter a read lock on a <see cref="ReaderWriterLockSlim"/>. Dispose the returned value to exit the read lock.
+    /// </summary>
+    /// <remarks>
+    /// This is intended to be used with a <see langword="using" /> statement or block.
+    /// </remarks>
+    [MustUseReturnValue]
+    public static RWReadGuard ReadGuard(this ReaderWriterLockSlim rwLock)
+    {
+        rwLock.EnterReadLock();
+        return new RWReadGuard(rwLock);
+    }
+
+    /// <summary>
+    /// Enter a write lock on a <see cref="ReaderWriterLockSlim"/>. Dispose the returned value to exit the write lock.
+    /// </summary>
+    /// <remarks>
+    /// This is intended to be used with a <see langword="using" /> statement or block.
+    /// </remarks>
+    [MustUseReturnValue]
+    public static RWWriteGuard WriteGuard(this ReaderWriterLockSlim rwLock)
+    {
+        rwLock.EnterWriteLock();
+        return new RWWriteGuard(rwLock);
+    }
+
+    /// <summary>
+    /// Wait on a <see cref="SemaphoreSlim"/>. Dispose the returned value to release.
+    /// </summary>
+    /// <remarks>
+    /// This is intended to be used with a <see langword="using" /> statement or block.
+    /// </remarks>
+    [MustUseReturnValue]
+    public static SemaphoreGuard WaitGuard(this SemaphoreSlim semaphore)
+    {
+        semaphore.Wait();
+        return new SemaphoreGuard(semaphore);
+    }
+
+    /// <summary>
+    /// Wait on a <see cref="SemaphoreSlim"/> asynchronously. Dispose the returned value to release.
+    /// </summary>
+    /// <remarks>
+    /// This is intended to be used with a <see langword="using" /> statement or block.
+    /// </remarks>
+    [MustUseReturnValue]
+    public static async ValueTask<SemaphoreGuard> WaitGuardAsync(this SemaphoreSlim semaphore)
+    {
+        await semaphore.WaitAsync();
+        return new SemaphoreGuard(semaphore);
+    }
+
+    // ReSharper disable once InconsistentNaming
+    public struct RWReadGuard : IDisposable
+    {
+        public readonly ReaderWriterLockSlim RwLock;
+        public bool Disposed { get; private set; }
+
+        public RWReadGuard(ReaderWriterLockSlim rwLock)
+        {
+            RwLock = rwLock;
+            Disposed = false;
+        }
+
+        public void Dispose()
+        {
+            if (Disposed)
+                throw new InvalidOperationException($"Double dispose of {nameof(RWReadGuard)}");
+
+            Disposed = true;
+            RwLock.ExitReadLock();
+        }
+    }
+
+    // ReSharper disable once InconsistentNaming
+    public struct RWWriteGuard : IDisposable
+    {
+        public readonly ReaderWriterLockSlim RwLock;
+        public bool Disposed { get; private set; }
+
+        public RWWriteGuard(ReaderWriterLockSlim rwLock)
+        {
+            RwLock = rwLock;
+            Disposed = false;
+        }
+
+        public void Dispose()
+        {
+            if (Disposed)
+                throw new InvalidOperationException($"Double dispose of {nameof(RWWriteGuard)}");
+
+            Disposed = true;
+            RwLock.ExitWriteLock();
+        }
+    }
+
+    public struct SemaphoreGuard : IDisposable
+    {
+        public readonly SemaphoreSlim Semaphore;
+        public bool Disposed { get; private set; }
+
+        public SemaphoreGuard(SemaphoreSlim semaphore)
+        {
+            Semaphore = semaphore;
+            Disposed = false;
+        }
+
+        public void Dispose()
+        {
+            if (Disposed)
+                throw new InvalidOperationException($"Double dispose of {nameof(SemaphoreGuard)}");
+
+            Disposed = true;
+            Semaphore.Release();
+        }
+    }
+}
+

Robust.Shared/Utility/Parse.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Globalization;
+
+namespace Robust.Shared.Utility;
+
+/// <summary>
+/// Helpers for parsing culture-invariant data.
+/// </summary>
+/// <remarks>
+/// APIs like <see cref="System.Int32.TryParse(string, out int)"/> are culture sensitive by default,
+/// and making them not culture sensitive is extremely verbose.
+/// These helpers are culture insensitive without requiring you to write a whole shakespeare novel
+/// for the privilege of having code that isn't gonna break for French people.</remarks>
+public static class Parse
+{
+    // INT32
+
+    public static bool TryInt32(ReadOnlySpan<char> text, out int result)
+    {
+        return TryInt32(text, NumberStyles.Integer, out result);
+    }
+
+    public static bool TryInt32(ReadOnlySpan<char> text, NumberStyles style, out int result)
+    {
+        return int.TryParse(text, style, CultureInfo.InvariantCulture, out result);
+    }
+
+    public static int Int32(ReadOnlySpan<char> text, NumberStyles style = NumberStyles.Integer)
+    {
+        return int.Parse(text, style, CultureInfo.InvariantCulture);
+    }
+
+    // INT64
+
+    public static bool TryInt64(ReadOnlySpan<char> text, out long result)
+    {
+        return TryInt64(text, NumberStyles.Integer, out result);
+    }
+
+    public static bool TryInt64(ReadOnlySpan<char> text, NumberStyles style, out long result)
+    {
+        return long.TryParse(text, style, CultureInfo.InvariantCulture, out result);
+    }
+
+    public static long Int64(ReadOnlySpan<char> text, NumberStyles style = NumberStyles.Integer)
+    {
+        return long.Parse(text, style, CultureInfo.InvariantCulture);
+    }
+
+    // FLOAT
+
+    public static bool TryFloat(ReadOnlySpan<char> text, out float result)
+    {
+        return TryFloat(text, NumberStyles.Float, out result);
+    }
+
+    public static bool TryFloat(ReadOnlySpan<char> text, NumberStyles style, out float result)
+    {
+        return float.TryParse(text, style, CultureInfo.InvariantCulture, out result);
+    }
+
+    public static float Float(ReadOnlySpan<char> text, NumberStyles style = NumberStyles.Float)
+    {
+        return float.Parse(text, style, CultureInfo.InvariantCulture);
+    }
+
+    // DOUBLE
+
+    public static bool TryDouble(ReadOnlySpan<char> text, out double result)
+    {
+        return TryDouble(text, NumberStyles.Float, out result);
+    }
+
+    public static bool TryDouble(ReadOnlySpan<char> text, NumberStyles style, out double result)
+    {
+        return double.TryParse(text, style, CultureInfo.InvariantCulture, out result);
+    }
+
+    public static double Double(ReadOnlySpan<char> text, NumberStyles style = NumberStyles.Float)
+    {
+        return double.Parse(text, style, CultureInfo.InvariantCulture);
+    }
+}

Robust.Shared/Utility/PoolHelpers.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Buffers;
+
+namespace Robust.Shared.Utility;
+
+/// <summary>
+/// Helpers for working with memory pooling such as <see cref="ArrayPool{T}"/>
+/// </summary>
+public static class PoolHelpers
+{
+    /// <summary>
+    /// Provides a disposable guard to return an array pool entry.
+    /// </summary>
+    /// <remarks>
+    /// This is intended to be used with using statements.
+    /// </remarks>
+    public static PoolReturnGuard<T> ReturnGuard<T>(this ArrayPool<T> pool, T[] buf)
+    {
+        return new PoolReturnGuard<T>(pool, buf);
+    }
+
+    /// <summary>
+    /// Disposes the given array into the given array pool on dispose.
+    /// </summary>
+    public readonly struct PoolReturnGuard<T> : IDisposable
+    {
+        private readonly ArrayPool<T> _pool;
+        private readonly T[] _array;
+
+        public PoolReturnGuard(ArrayPool<T> pool, T[] array)
+        {
+            _pool = pool;
+            _array = array;
+        }
+
+        public void Dispose()
+        {
+            _pool.Return(_array);
+        }
+    }
+}

Robust.Shared/Utility/ZStd.cs

@@ -0,0 +1,429 @@
+using System;
+using System.Buffers;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Runtime.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using SharpZstd.Interop;
+using static SharpZstd.Interop.Zstd;
+
+namespace Robust.Shared.Utility;
+
+public static class ZStd
+{
+    public static int CompressBound(int length)
+    {
+        return (int)ZSTD_COMPRESSBOUND((nuint)length);
+    }
+
+    public static unsafe int Compress(
+        Span<byte> into,
+        ReadOnlySpan<byte> data,
+        int compressionLevel = ZSTD_CLEVEL_DEFAULT)
+    {
+        fixed (byte* dst = into)
+        fixed (byte* src = data)
+        {
+            var result = ZSTD_compress(dst, (nuint) into.Length, src, (nuint) data.Length, compressionLevel);
+            ZStdException.ThrowIfError(result);
+            return (int) result;
+        }
+    }
+}
+
+[Serializable]
+internal sealed class ZStdException : Exception
+{
+    public ZStdException()
+    {
+    }
+
+    public ZStdException(string message) : base(message)
+    {
+    }
+
+    public ZStdException(string message, Exception inner) : base(message, inner)
+    {
+    }
+
+    public static unsafe ZStdException FromCode(nuint code)
+    {
+        return new ZStdException(Marshal.PtrToStringUTF8((IntPtr)ZSTD_getErrorName(code))!);
+    }
+
+    public static void ThrowIfError(nuint code)
+    {
+        if (ZSTD_isError(code) != 0)
+            throw FromCode(code);
+    }
+}
+
+public sealed unsafe class ZStdCompressionContext : IDisposable
+{
+    public ZSTD_CCtx* Context { get; private set; }
+
+    private bool Disposed => Context == null;
+
+    public ZStdCompressionContext()
+    {
+        Context = ZSTD_createCCtx();
+    }
+
+    public void SetParameter(ZSTD_cParameter parameter, int value)
+    {
+        CheckDisposed();
+
+        ZSTD_CCtx_setParameter(Context, parameter, value);
+    }
+
+    public int Compress(Span<byte> destination, Span<byte> source, int compressionLevel = ZSTD_CLEVEL_DEFAULT)
+    {
+        CheckDisposed();
+
+        fixed (byte* dst = destination)
+        fixed (byte* src = source)
+        {
+            var ret = ZSTD_compressCCtx(
+                Context,
+                dst, (nuint)destination.Length,
+                src, (nuint)source.Length,
+                compressionLevel);
+
+            ZStdException.ThrowIfError(ret);
+            return (int)ret;
+        }
+    }
+
+    ~ZStdCompressionContext()
+    {
+        Dispose();
+    }
+
+    public void Dispose()
+    {
+        if (Disposed)
+            return;
+
+        ZSTD_freeCCtx(Context);
+        Context = null;
+        GC.SuppressFinalize(this);
+    }
+
+    private void CheckDisposed()
+    {
+        if (Disposed)
+            throw new ObjectDisposedException(nameof(ZStdCompressionContext));
+    }
+}
+
+internal sealed class ZStdDecompressStream : Stream
+{
+    private readonly Stream _baseStream;
+    private readonly bool _ownStream;
+    private readonly unsafe ZSTD_DCtx* _ctx;
+    private readonly byte[] _buffer;
+    private int _bufferPos;
+    private int _bufferSize;
+    private bool _disposed;
+
+    public unsafe ZStdDecompressStream(Stream baseStream, bool ownStream = true)
+    {
+        _baseStream = baseStream;
+        _ownStream = ownStream;
+        _ctx = ZSTD_createDCtx();
+        _buffer = ArrayPool<byte>.Shared.Rent((int)ZSTD_DStreamInSize());
+    }
+
+    protected override unsafe void Dispose(bool disposing)
+    {
+        if (_disposed)
+            return;
+
+        _disposed = true;
+        ZSTD_freeDCtx(_ctx);
+
+        if (disposing)
+        {
+            if (_ownStream)
+                _baseStream.Dispose();
+
+            ArrayPool<byte>.Shared.Return(_buffer);
+        }
+    }
+
+    public override void Flush()
+    {
+        ThrowIfDisposed();
+        _baseStream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        return Read(buffer.AsSpan(offset, count));
+    }
+
+    public override int ReadByte()
+    {
+        Span<byte> buf = stackalloc byte[1];
+        return Read(buf) == 0 ? -1 : buf[0];
+    }
+
+    public override unsafe int Read(Span<byte> buffer)
+    {
+        ThrowIfDisposed();
+        do
+        {
+            if (_bufferSize == 0 || _bufferPos == _bufferSize)
+            {
+                _bufferPos = 0;
+                _bufferSize = _baseStream.Read(_buffer);
+
+                if (_bufferSize == 0)
+                    return 0;
+            }
+
+            fixed (byte* inputPtr = _buffer)
+            fixed (byte* outputPtr = buffer)
+            {
+                var outputBuf = new ZSTD_outBuffer { dst = outputPtr, pos = 0, size = (nuint)buffer.Length };
+                var inputBuf = new ZSTD_inBuffer { src = inputPtr, pos = (nuint)_bufferPos, size = (nuint)_bufferSize };
+                var ret = ZSTD_decompressStream(_ctx, &outputBuf, &inputBuf);
+
+                _bufferPos = (int)inputBuf.pos;
+                ZStdException.ThrowIfError(ret);
+
+                if (outputBuf.pos > 0)
+                    return (int)outputBuf.pos;
+            }
+        } while (true);
+    }
+
+    public override async ValueTask<int> ReadAsync(
+        Memory<byte> buffer,
+        CancellationToken cancellationToken = default)
+    {
+        ThrowIfDisposed();
+        do
+        {
+            if (_bufferSize == 0 || _bufferPos == _bufferSize)
+            {
+                _bufferPos = 0;
+                _bufferSize = await _baseStream.ReadAsync(_buffer, cancellationToken);
+
+                if (_bufferSize == 0)
+                    return 0;
+            }
+
+            unsafe
+            {
+                fixed (byte* inputPtr = _buffer)
+                fixed (byte* outputPtr = buffer.Span)
+                {
+                    ZSTD_outBuffer outputBuf = default;
+                    outputBuf.dst = outputPtr;
+                    outputBuf.pos = 0;
+                    outputBuf.size = (nuint)buffer.Length;
+                    ZSTD_inBuffer inputBuf = default;
+                    inputBuf.src = inputPtr;
+                    inputBuf.pos = (nuint)_bufferPos;
+                    inputBuf.size = (nuint)_bufferSize;
+
+                    var ret = ZSTD_decompressStream(_ctx, &outputBuf, &inputBuf);
+
+                    _bufferPos = (int)inputBuf.pos;
+                    ZStdException.ThrowIfError(ret);
+
+                    if (outputBuf.pos > 0)
+                        return (int)outputBuf.pos;
+                }
+            }
+        } while (true);
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override void SetLength(long value)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override bool CanRead => true;
+    public override bool CanSeek => false;
+    public override bool CanWrite => false;
+    public override long Length => throw new NotSupportedException();
+
+    public override long Position
+    {
+        get => throw new NotSupportedException();
+        set => throw new NotSupportedException();
+    }
+
+    private void ThrowIfDisposed()
+    {
+        if (_disposed)
+            throw new ObjectDisposedException(nameof(ZStdDecompressStream));
+    }
+}
+
+internal sealed class ZStdCompressStream : Stream
+{
+    private readonly Stream _baseStream;
+    private readonly bool _ownStream;
+    public ZStdCompressionContext Context { get; }
+    private readonly byte[] _buffer;
+    private int _bufferPos;
+    private bool _disposed;
+    private bool _hasSession;
+
+    public ZStdCompressStream(Stream baseStream, bool ownStream = true)
+    {
+        Context = new ZStdCompressionContext();
+        _baseStream = baseStream;
+        _ownStream = ownStream;
+        _buffer = ArrayPool<byte>.Shared.Rent((int)ZSTD_CStreamOutSize());
+    }
+
+    public override void Flush()
+    {
+        FlushInternal(ZSTD_EndDirective.ZSTD_e_flush);
+    }
+
+    public void FlushEnd()
+    {
+        _hasSession = false;
+        FlushInternal(ZSTD_EndDirective.ZSTD_e_end);
+    }
+
+    private unsafe void FlushInternal(ZSTD_EndDirective directive)
+    {
+        fixed (byte* outPtr = _buffer)
+        {
+            ZSTD_outBuffer outBuf = default;
+            outBuf.size = (nuint)_buffer.Length;
+            outBuf.pos = (nuint)_bufferPos;
+            outBuf.dst = outPtr;
+
+            ZSTD_inBuffer inBuf;
+
+            while (true)
+            {
+                var err = ZSTD_compressStream2(Context.Context, &outBuf, &inBuf, directive);
+                ZStdException.ThrowIfError(err);
+                _bufferPos = (int)outBuf.pos;
+
+                _baseStream.Write(_buffer.AsSpan(0, (int)outBuf.pos));
+                _bufferPos = 0;
+                outBuf.pos = 0;
+
+                if (err == 0)
+                    break;
+            }
+        }
+
+        _baseStream.Flush();
+    }
+
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override void SetLength(long value)
+    {
+        throw new NotSupportedException();
+    }
+
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        Write(buffer.AsSpan(offset, count));
+    }
+
+    public override unsafe void Write(ReadOnlySpan<byte> buffer)
+    {
+        ThrowIfDisposed();
+
+        _hasSession = true;
+
+        fixed (byte* outPtr = _buffer)
+        fixed (byte* inPtr = buffer)
+        {
+            ZSTD_outBuffer outBuf = default;
+            outBuf.size = (nuint)_buffer.Length;
+            outBuf.pos = (nuint)_bufferPos;
+            outBuf.dst = outPtr;
+
+            ZSTD_inBuffer inBuf = default;
+            inBuf.pos = 0;
+            inBuf.size = (nuint)buffer.Length;
+            inBuf.src = inPtr;
+
+            while (true)
+            {
+                var err = ZSTD_compressStream2(Context.Context, &outBuf, &inBuf, ZSTD_EndDirective.ZSTD_e_continue);
+                ZStdException.ThrowIfError(err);
+                _bufferPos = (int)outBuf.pos;
+
+                if (inBuf.pos >= inBuf.size)
+                    break;
+
+                // Not all input data consumed. Flush output buffer and continue.
+                _baseStream.Write(_buffer.AsSpan(0, (int)outBuf.pos));
+                _bufferPos = 0;
+                outBuf.pos = 0;
+            }
+        }
+    }
+
+    public override bool CanRead => false;
+    public override bool CanSeek => false;
+    public override bool CanWrite => true;
+    public override long Length => throw new NotSupportedException();
+
+    public override long Position
+    {
+        get => throw new NotSupportedException();
+        set => throw new NotSupportedException();
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+
+        if (_disposed)
+            return;
+
+        if (disposing)
+        {
+            if (_hasSession)
+                FlushEnd();
+
+            if (_ownStream)
+                _baseStream.Dispose();
+
+            ArrayPool<byte>.Shared.Return(_buffer);
+            Context.Dispose();
+        }
+
+        _disposed = true;
+    }
+
+    private void ThrowIfDisposed()
+    {
+        if (_disposed)
+            throw new ObjectDisposedException(nameof(ZStdCompressStream));
+    }
+}