Add support for including map/game saves in replays. (#6189)

* Improve map serialization error logging

* Prevent remove children of erroring entities

* better logging

* Improve error tolerance

* Even more exception tolerance

* missing !

* Add WriteYaml and WriteObject to IReplayFileWriter

* Add MapLoaderSystem.TrySaveAllEntities()

* On second thought, WriteObject will just be abused

* I forgot to commit

* Add default implementation to avoid breaking changes

* release notes

* fix merge issues

---------

Co-authored-by: metalgearsloth <31366439+metalgearsloth@users.noreply.github.com>

RELEASE-NOTES.md

@@ -39,12 +39,13 @@ END TEMPLATE-->
 
 ### New features
 
-*None yet*
+* Added `IReplayFileWriter.WriteYaml()`, for writing yaml documents to a replay zip file.
 
 ### Bugfixes
 
-*None yet*
-
+* `ActorComponent` now has the `UnsavedComponentAttribute`
+  * Previously it was unintentionally get serialized to yaml, which could result in NREs when deserializing.
+  
 ### Other
 
 *None yet*

Resources/EnginePrototypes/Audio/audio_entities.yml

@@ -2,7 +2,7 @@
   id: Audio
   name: Audio
   description: Audio entity used by engine
-  save: false
+  save: false # TODO PERSISTENCE what about looping or long sounds?
   components:
     - type: Transform
       gridTraversal: false

Robust.Shared/EntitySerialization/EntitySerializer.cs

@@ -62,6 +62,7 @@ public sealed class EntitySerializer : ISerializationContext,
     private readonly ISawmill _log;
     public readonly Dictionary<EntityUid, int> YamlUidMap = new();
     public readonly HashSet<int> YamlIds = new();
+    public readonly ValueDataNode InvalidNode = new("invalid");
 
     public string? CurrentComponent { get; private set; }
     public Entity<MetaDataComponent>? CurrentEntity { get; private set; }
@@ -222,6 +223,7 @@ public sealed class EntitySerializer : ISerializationContext,
     /// setting of <see cref="SerializationOptions.MissingEntityBehaviour"/> it may auto-include additional entities
     /// aside from the one provided.
     /// </summary>
+    /// <param name="entities">The set of entities to serialize</param>
     public void SerializeEntities(HashSet<EntityUid> entities)
     {
         foreach (var uid in entities)
@@ -329,7 +331,12 @@ public sealed class EntitySerializer : ISerializationContext,
             return true;
         }
 
-        // iterate over all of its children and grab the first grid with a mapping
+        map = null;
+
+        // if this is a map, iterate over all of its children and grab the first grid with a mapping
+        if (!_mapQuery.HasComponent(root))
+            return false;
+
         var xform = _xformQuery.GetComponent(root);
         foreach (var child in xform._children)
         {
@@ -339,7 +346,6 @@ public sealed class EntitySerializer : ISerializationContext,
             return true;
         }
 
-        map = null;
         return false;
     }
 
@@ -979,7 +985,7 @@ public sealed class EntitySerializer : ISerializationContext,
         if (CurrentComponent == _xformName)
         {
             if (value == EntityUid.Invalid)
-                return new ValueDataNode("invalid");
+                return InvalidNode;
 
             DebugTools.Assert(!Orphans.Contains(CurrentEntityYamlUid));
             Orphans.Add(CurrentEntityYamlUid);
@@ -987,13 +993,13 @@ public sealed class EntitySerializer : ISerializationContext,
             if (Options.ErrorOnOrphan && CurrentEntity != null && value != Truncate && !ErroringEntities.Contains(value))
                 _log.Error($"Serializing entity {EntMan.ToPrettyString(CurrentEntity)} without including its parent {EntMan.ToPrettyString(value)}");
 
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (ErroringEntities.Contains(value))
         {
             // Referenced entity already logged an error, so we just silently fail.
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (value == EntityUid.Invalid)
@@ -1001,7 +1007,7 @@ public sealed class EntitySerializer : ISerializationContext,
             if (Options.MissingEntityBehaviour != MissingEntityBehaviour.Ignore)
                 _log.Error($"Encountered an invalid entityUid reference.");
 
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (value == Truncate)
@@ -1016,9 +1022,9 @@ public sealed class EntitySerializer : ISerializationContext,
                 _log.Error(EntMan.Deleted(value)
                     ? $"Encountered a reference to a deleted entity {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}."
                     : $"Encountered a reference to a missing entity: {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}.");
-                return new ValueDataNode("invalid");
+                return InvalidNode;
             case MissingEntityBehaviour.Ignore:
-                return new ValueDataNode("invalid");
+                return InvalidNode;
             case MissingEntityBehaviour.IncludeNullspace:
                 if (!EntMan.TryGetComponent(value, out TransformComponent? xform)
                     || xform.ParentUid != EntityUid.Invalid

Robust.Shared/EntitySerialization/SerializationEnums.cs

@@ -87,7 +87,6 @@ public enum MissingEntityBehaviour
     AutoInclude,
 }
 
-
 public enum EntityExceptionBehaviour
 {
     /// <summary>

Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Robust.Shared.GameObjects;
 using Robust.Shared.Map;
@@ -18,6 +19,10 @@ public sealed partial class MapLoaderSystem
     /// <summary>
     /// Recursively serialize the given entities and all of their children.
     /// </summary>
+    /// <remarks>
+    /// This method is not optimized for being given a large set of entities. I.e., this should be a small handful of
+    /// maps or grids, not something like <see cref="EntityManager.AllEntityUids"/>.
+    /// </remarks>
     public (MappingDataNode Node, FileCategory Category) SerializeEntitiesRecursive(
         HashSet<EntityUid> entities,
         SerializationOptions? options = null)
@@ -29,8 +34,6 @@ public sealed partial class MapLoaderSystem
         Log.Info($"Serializing entities: {string.Join(", ", entities.Select(x => ToPrettyString(x).ToString()))}");
 
         var maps = entities.Select(x => Transform(x).MapID).ToHashSet();
-        var ev = new BeforeSerializationEvent(entities, maps);
-        RaiseLocalEvent(ev);
 
         // In case no options were provided, we assume that if all of the starting entities are pre-init, we should
         // expect that **all** entities that get serialized should be pre-init.
@@ -39,6 +42,9 @@ public sealed partial class MapLoaderSystem
             ExpectPreInit = (entities.All(x => LifeStage(x) < EntityLifeStage.MapInitialized))
         };
 
+        var ev = new BeforeSerializationEvent(entities, maps, opts.Category);
+        RaiseLocalEvent(ev);
+
         var serializer = new EntitySerializer(_dependency, opts);
         serializer.OnIsSerializeable += OnIsSerializable;
         serializer.SerializeEntityRecursive(entities);
@@ -230,4 +236,89 @@ public sealed partial class MapLoaderSystem
         Write(path, data);
         return true;
     }
+
+    /// <inheritdoc cref="TrySerializeAllEntities(out MappingDataNode, SerializationOptions?)"/>
+    public bool TrySaveAllEntities(ResPath path, SerializationOptions? options = null)
+    {
+        if (!TrySerializeAllEntities(out var data, options))
+            return false;
+
+        Write(path, data);
+        return true;
+    }
+
+    /// <summary>
+    /// Attempt to serialize all entities.
+    /// </summary>
+    /// <remarks>
+    /// Note that this alone is not sufficient for a proper full-game save, as the game may contain things like chat
+    /// logs or resources and prototypes that were uploaded mid-game.
+    /// </remarks>
+    public bool TrySerializeAllEntities([NotNullWhen(true)] out MappingDataNode? data, SerializationOptions? options = null)
+    {
+        data  = null;
+        var opts = options ?? SerializationOptions.Default with
+        {
+            MissingEntityBehaviour = MissingEntityBehaviour.Error
+        };
+
+        opts.Category = FileCategory.Save;
+        _stopwatch.Restart();
+        Log.Info($"Serializing all entities");
+
+        var entities = EntityManager.GetEntities().ToHashSet();
+        var maps = _mapSystem.Maps.Keys.ToHashSet();
+        var ev = new BeforeSerializationEvent(entities, maps, FileCategory.Save);
+        var serializer = new EntitySerializer(_dependency, opts);
+
+        // Remove any non-serializable entities and their children (prevent error spam)
+        var toRemove = new Queue<EntityUid>();
+        foreach (var entity in entities)
+        {
+            // TODO SERIALIZATION Perf
+            // IsSerializable gets called again by serializer.SerializeEntities()
+            if (!serializer.IsSerializable(entity))
+                toRemove.Enqueue(entity);
+        }
+
+        if (toRemove.Count > 0)
+        {
+            if (opts.MissingEntityBehaviour == MissingEntityBehaviour.Error)
+            {
+                // The save will probably contain references to the non-serializable entities, and we avoid spamming errors.
+                opts.MissingEntityBehaviour = MissingEntityBehaviour.Ignore;
+                Log.Error($"Attempted to serialize one or more non-serializable entities");
+            }
+
+            while (toRemove.TryDequeue(out var next))
+            {
+                entities.Remove(next);
+                foreach (var uid in Transform(next)._children)
+                {
+                    toRemove.Enqueue(uid);
+                }
+            }
+        }
+
+        try
+        {
+            RaiseLocalEvent(ev);
+            serializer.OnIsSerializeable += OnIsSerializable;
+            serializer.SerializeEntities(entities);
+            data = serializer.Write();
+            var cat = serializer.GetCategory();
+            DebugTools.AssertEqual(cat, FileCategory.Save);
+            var ev2 = new AfterSerializationEvent(entities, data, cat);
+            RaiseLocalEvent(ev2);
+
+            Log.Debug($"Serialized {serializer.EntityData.Count} entities in {_stopwatch.Elapsed}");
+        }
+        catch (Exception e)
+        {
+            Log.Error($"Caught exception while trying to serialize all entities:\n{e}");
+            return false;
+        }
+
+        return true;
+    }
 }

Robust.Shared/Map/Events/MapSerializationEvents.cs

@@ -34,7 +34,10 @@ public sealed class BeforeEntityReadEvent
 /// For convenience, the event also contains a set with all the maps that the entities are on. This does not
 /// necessarily mean that the maps are themselves getting serialized.
 /// </summary>
-public readonly record struct BeforeSerializationEvent(HashSet<EntityUid> Entities, HashSet<MapId> MapIds);
+public readonly record struct BeforeSerializationEvent(
+    HashSet<EntityUid> Entities,
+    HashSet<MapId> MapIds,
+    FileCategory Category = FileCategory.Unknown);
 
 /// <summary>
 /// This event is broadcast just after entities (and their children) have been serialized, but before it gets written to a yaml file.

Robust.Shared/Player/ActorComponent.cs

@@ -3,7 +3,7 @@ using Robust.Shared.ViewVariables;
 
 namespace Robust.Shared.Player;
 
-[RegisterComponent]
+[RegisterComponent, UnsavedComponent]
 public sealed partial class ActorComponent : Component
 {
     [ViewVariables]

Robust.Shared/Replays/IReplayRecordingManager.cs

@@ -2,11 +2,15 @@ using Robust.Shared.GameObjects;
 using Robust.Shared.Serialization.Markdown.Mapping;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.IO.Compression;
 using System.Threading.Tasks;
 using Robust.Shared.ContentPack;
 using Robust.Shared.GameStates;
+using Robust.Shared.Serialization;
 using Robust.Shared.Utility;
+using YamlDotNet.Core;
+using YamlDotNet.RepresentationModel;
 
 namespace Robust.Shared.Replays;
 
@@ -202,6 +206,25 @@ public interface IReplayFileWriter
         ResPath path,
         ReadOnlyMemory<byte> bytes,
         CompressionLevel compressionLevel = CompressionLevel.Optimal);
+
+    /// <summary>
+    /// Writes a yaml document into a file in the replay.
+    /// </summary>
+    /// <param name="path">The file path to write to.</param>
+    /// <param name="yaml">The yaml document to write to the file.</param>
+    /// <param name="compressionLevel">How much to compress the file.</param>
+    void WriteYaml(
+        ResPath path,
+        YamlDocument yaml,
+        CompressionLevel compressionLevel = CompressionLevel.Optimal)
+    {
+        var memStream = new MemoryStream();
+        using var writer = new StreamWriter(memStream);
+        var yamlStream = new YamlStream {yaml};
+        yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false);
+        writer.Flush();
+        WriteBytes(path, memStream.AsMemory(), compressionLevel);
+    }
 }
 
 /// <summary>

Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs

@@ -26,22 +26,30 @@ internal abstract partial class SharedReplayRecordingManager
     // and even then not for much longer than a couple hundred ms at most.
     private readonly List<Task> _finalizingWriteTasks = new();
 
-    private void WriteYaml(RecordingState state, ResPath path, YamlDocument data)
+    private void WriteYaml(
+        RecordingState state,
+        ResPath path,
+        YamlDocument data,
+        CompressionLevel level = CompressionLevel.Optimal)
     {
         var memStream = new MemoryStream();
         using var writer = new StreamWriter(memStream);
         var yamlStream = new YamlStream { data };
         yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false);
         writer.Flush();
-        WriteBytes(state, path, memStream.AsMemory());
+        WriteBytes(state, path, memStream.AsMemory(), level);
     }
 
-    private void WriteSerializer<T>(RecordingState state, ResPath path, T obj)
+    private void WriteSerializer<T>(
+        RecordingState state,
+        ResPath path,
+        T obj,
+        CompressionLevel level = CompressionLevel.Optimal)
     {
         var memStream = new MemoryStream();
         _serializer.SerializeDirect(memStream, obj);
 
-        WriteBytes(state, path, memStream.AsMemory());
+        WriteBytes(state, path, memStream.AsMemory(), level);
     }
 
     private void WritePooledBytes(

Robust.Shared/Replays/SharedReplayRecordingManager.cs

@@ -375,6 +375,11 @@ internal abstract partial class SharedReplayRecordingManager : IReplayRecordingM
     private void WriteFinalMetadata(RecordingState recState)
     {
         var yamlMetadata = new MappingDataNode();
+
+        // TODO REPLAYS
+        // Why are these separate events?
+        // I assume it was for backwards compatibility / avoiding breaking changes?
+        // But eventually RecordingStopped2 will probably be renamed and there'll just be more breaking changes.
         RecordingStopped?.Invoke(yamlMetadata);
         RecordingStopped2?.Invoke(new ReplayRecordingStopped
         {
@@ -552,6 +557,12 @@ internal abstract partial class SharedReplayRecordingManager : IReplayRecordingM
             manager.WriteBytes(state, path, bytes, compressionLevel);
         }
 
+        void IReplayFileWriter.WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel)
+        {
+            CheckDisposed();
+            manager.WriteYaml(state, path, document, compressionLevel);
+        }
+
         private void CheckDisposed()
         {
             if (state.Done)