Custom YAML -> DataNode parser. (#3496)

Robust.Shared/Prototypes/PrototypeManager.cs

@@ -16,6 +16,7 @@ using Robust.Shared.Serialization.Manager;
 using Robust.Shared.Serialization.Manager.Attributes;
 using Robust.Shared.Serialization.Markdown;
 using Robust.Shared.Serialization.Markdown.Mapping;
+using Robust.Shared.Serialization.Markdown.Sequence;
 using Robust.Shared.Serialization.Markdown.Validation;
 using Robust.Shared.Serialization.Markdown.Value;
 using Robust.Shared.Utility;
@@ -549,8 +550,9 @@ namespace Robust.Shared.Prototypes
         public void LoadDirectory(ResourcePath path, bool overwrite = false, Dictionary<Type, HashSet<string>>? changed = null)
         {
             _hasEverBeenReloaded = true;
-            var streams = Resources.ContentFindFiles(path).ToList().AsParallel()
-                .Where(filePath => filePath.Extension == "yml" && !filePath.Filename.StartsWith("."));
+            var streams = Resources.ContentFindFiles(path)
+                .Where(filePath => filePath.Extension == "yml" && !filePath.Filename.StartsWith("."))
+                .ToArray();
 
             foreach (var resourcePath in streams)
             {
@@ -645,34 +647,79 @@ namespace Robust.Shared.Prototypes
                 using var reader = ReadFile(file, !overwrite);
 
                 if (reader == null)
-                {
                     return;
-                }
 
-                var yamlStream = new YamlStream();
-                yamlStream.Load(reader);
+                // LoadedData?.Invoke(yamlStream, file.ToString());
 
-                LoadedData?.Invoke(yamlStream, file.ToString());
-
-                for (var i = 0; i < yamlStream.Documents.Count; i++)
+                var i = 0;
+                foreach (var document in DataNodeParser.ParseYamlStream(reader))
                 {
                     try
                     {
-                        LoadFromDocument(yamlStream.Documents[i], overwrite, changed);
+                        var seq = (SequenceDataNode)document.Root;
+                        foreach (var mapping in seq.Sequence)
+                        {
+                            LoadFromMapping((MappingDataNode) mapping, overwrite, changed);
+                        }
                     }
                     catch (Exception e)
                     {
                         Logger.ErrorS("eng", $"Exception whilst loading prototypes from {file}#{i}:\n{e}");
                     }
+
+                    i += 1;
                 }
             }
-            catch (YamlException e)
+            catch (Exception e)
             {
                 var sawmill = Logger.GetSawmill("eng");
                 sawmill.Error("YamlException whilst loading prototypes from {0}: {1}", file, e.Message);
             }
         }
 
+        private void LoadFromMapping(
+            MappingDataNode datanode,
+            bool overwrite = false,
+            Dictionary<Type, HashSet<string>>? changed = null)
+        {
+            var type = datanode.Get<ValueDataNode>("type").Value;
+            if (!_prototypeTypes.TryGetValue(type, out var prototypeType))
+            {
+                if (_ignoredPrototypeTypes.Contains(type))
+                    return;
+
+                throw new PrototypeLoadException($"Unknown prototype type: '{type}'");
+            }
+
+            if (!datanode.TryGet<ValueDataNode>(IdDataFieldAttribute.Name, out var idNode))
+                throw new PrototypeLoadException($"Prototype type {type} is missing an 'id' datafield.");
+
+            if (!overwrite && _prototypeResults[prototypeType].ContainsKey(idNode.Value))
+                throw new PrototypeLoadException($"Duplicate ID: '{idNode.Value}'");
+
+            _prototypeResults[prototypeType][idNode.Value] = datanode;
+            if (prototypeType.IsAssignableTo(typeof(IInheritingPrototype)))
+            {
+                if (datanode.TryGet(ParentDataFieldAttribute.Name, out var parentNode))
+                {
+                    var parents = _serializationManager.Read<string[]>(parentNode);
+                    _inheritanceTrees[prototypeType].Add(idNode.Value, parents);
+                }
+                else
+                {
+                    _inheritanceTrees[prototypeType].Add(idNode.Value);
+                }
+            }
+
+            if (changed == null)
+                return;
+
+            if (!changed.TryGetValue(prototypeType, out var set))
+                changed[prototypeType] = set = new HashSet<string>();
+
+            set.Add(idNode.Value);
+        }
+
         public void LoadFromStream(TextReader stream, bool overwrite = false, Dictionary<Type, HashSet<string>>? changed = null)
         {
             _hasEverBeenReloaded = true;
@@ -751,46 +798,7 @@ namespace Robust.Shared.Prototypes
             foreach (var node in rootNode.Cast<YamlMappingNode>())
             {
                 var datanode = node.ToDataNodeCast<MappingDataNode>();
-                var type = datanode.Get<ValueDataNode>("type").Value;
-                if (!_prototypeTypes.ContainsKey(type))
-                {
-                    if (_ignoredPrototypeTypes.Contains(type))
-                    {
-                        continue;
-                    }
-
-                    throw new PrototypeLoadException($"Unknown prototype type: '{type}'");
-                }
-
-                var prototypeType = _prototypeTypes[type];
-
-                if (!datanode.TryGet<ValueDataNode>(IdDataFieldAttribute.Name, out var idNode))
-                    throw new PrototypeLoadException($"Prototype type {type} is missing an 'id' datafield.");
-
-                if (!overwrite && _prototypeResults[prototypeType].ContainsKey(idNode.Value))
-                {
-                    throw new PrototypeLoadException($"Duplicate ID: '{idNode.Value}'");
-                }
-
-                _prototypeResults[prototypeType][idNode.Value] = datanode;
-                if (prototypeType.IsAssignableTo(typeof(IInheritingPrototype)))
-                {
-                    if (datanode.TryGet(ParentDataFieldAttribute.Name, out var parentNode))
-                    {
-                        var parents = _serializationManager.Read<string[]>(parentNode);
-                        _inheritanceTrees[prototypeType].Add(idNode.Value, parents);
-                    }
-                    else
-                    {
-                        _inheritanceTrees[prototypeType].Add(idNode.Value);
-                    }
-                }
-
-                if (changed == null) continue;
-
-                if (!changed.TryGetValue(prototypeType, out var set))
-                    changed[prototypeType] = set = new HashSet<string>();
-                set.Add(idNode.Value);
+                LoadFromMapping(datanode, overwrite, changed);
             }
         }
 

Robust.Shared/Serialization/Markdown/DataNodeParser.cs

@@ -0,0 +1,271 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Robust.Shared.Collections;
+using Robust.Shared.Serialization.Markdown.Mapping;
+using Robust.Shared.Serialization.Markdown.Sequence;
+using Robust.Shared.Serialization.Markdown.Value;
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+
+namespace Robust.Shared.Serialization.Markdown;
+
+// YDN has broken nullable annotations. Yeppers.
+#nullable disable
+
+public static class DataNodeParser
+{
+    public static IEnumerable<DataNodeDocument> ParseYamlStream(TextReader reader)
+    {
+        return ParseYamlStream(new Parser(reader));
+    }
+
+    internal static IEnumerable<DataNodeDocument> ParseYamlStream(Parser parser)
+    {
+        parser.Consume<StreamStart>();
+
+        while (!parser.TryConsume<StreamEnd>(out _))
+        {
+            yield return ParseDocument(parser);
+        }
+    }
+
+    private static DataNodeDocument ParseDocument(Parser parser)
+    {
+        var state = new DocumentState();
+
+        parser.Consume<DocumentStart>();
+
+        var root = Parse(parser, state);
+
+        parser.Consume<DocumentEnd>();
+
+        ResolveAliases(state);
+
+        return new DataNodeDocument(root);
+    }
+
+    private static DataNode Parse(Parser parser, DocumentState state)
+    {
+        if (parser.Current is Scalar)
+            return ParseValue(parser, state);
+
+        if (parser.Current is SequenceStart)
+            return ParseSequence(parser, state);
+
+        if (parser.Current is MappingStart)
+            return ParseMapping(parser, state);
+
+        if (parser.Current is AnchorAlias)
+            return ParseAlias(parser, state);
+
+        throw new NotSupportedException();
+    }
+
+    private static DataNode ParseAlias(Parser parser, DocumentState state)
+    {
+        var alias = parser.Consume<AnchorAlias>();
+
+        if (!state.Anchors.TryGetValue(alias.Value, out var node))
+        {
+            // Don't have this anchor yet. It may be defined later in the document.
+            return new DataNodeAlias(alias.Value);
+        }
+
+        return node;
+    }
+
+    private static ValueDataNode ParseValue(Parser parser, DocumentState state)
+    {
+        var ev = parser.Consume<Scalar>();
+        var node = new ValueDataNode(ev.Value);
+        node.Tag = ConvertTag(ev.Tag);
+        node.Start = ev.Start;
+        node.End = ev.End;
+
+        NodeParsed(node, ev, false, state);
+
+        return node;
+    }
+
+    private static SequenceDataNode ParseSequence(Parser parser, DocumentState state)
+    {
+        var ev = parser.Consume<SequenceStart>();
+
+        var node = new SequenceDataNode();
+        node.Tag = ConvertTag(ev.Tag);
+        node.Start = ev.Start;
+
+        var unresolvedAlias = false;
+
+        SequenceEnd seqEnd;
+        while (!parser.TryConsume(out seqEnd))
+        {
+            var value = Parse(parser, state);
+
+            node.Add(value);
+
+            unresolvedAlias |= value is DataNodeAlias;
+        }
+
+        node.End = seqEnd.End;
+
+        NodeParsed(node, ev, unresolvedAlias, state);
+
+        return node;
+    }
+
+    private static MappingDataNode ParseMapping(Parser parser, DocumentState state)
+    {
+        var ev = parser.Consume<MappingStart>();
+
+        var node = new MappingDataNode();
+        node.Tag = ConvertTag(ev.Tag);
+
+        var unresolvedAlias = false;
+
+        MappingEnd mapEnd;
+        while (!parser.TryConsume(out mapEnd))
+        {
+            var key = Parse(parser, state);
+            var value = Parse(parser, state);
+
+            node.Add(key, value);
+
+            unresolvedAlias |= key is DataNodeAlias;
+            unresolvedAlias |= value is DataNodeAlias;
+        }
+
+        node.End = mapEnd.End;
+
+        NodeParsed(node, ev, unresolvedAlias, state);
+
+        return node;
+    }
+
+    private static void NodeParsed(DataNode node, NodeEvent ev, bool unresolvedAlias, DocumentState state)
+    {
+        if (unresolvedAlias)
+            state.UnresolvedAliasOwners.Add(node);
+
+        if (ev.Anchor.IsEmpty)
+            return;
+
+        if (state.Anchors.ContainsKey(ev.Anchor))
+            throw new DataParseException($"Duplicate anchor defined in document: {ev.Anchor}");
+
+        state.Anchors[ev.Anchor] = node;
+    }
+
+    private static void ResolveAliases(DocumentState state)
+    {
+        foreach (var node in state.UnresolvedAliasOwners)
+        {
+            switch (node)
+            {
+                case MappingDataNode mapping:
+                    ResolveMappingAliases(mapping, state);
+                    break;
+
+                case SequenceDataNode sequence:
+                    ResolveSequenceAliases(sequence, state);
+                    break;
+            }
+        }
+    }
+
+    private static void ResolveMappingAliases(MappingDataNode mapping, DocumentState state)
+    {
+        var swaps = new ValueList<(DataNode key, DataNode value)>();
+
+        foreach (var (key, value) in mapping)
+        {
+            if (key is not DataNodeAlias && value is not DataNodeAlias)
+                return;
+
+            var newKey = key is DataNodeAlias keyAlias ? ResolveAlias(keyAlias, state) : key;
+            var newValue = value is DataNodeAlias valueAlias ? ResolveAlias(valueAlias, state) : value;
+
+            swaps.Add((newKey, newValue));
+            mapping.Remove(key);
+        }
+
+        foreach (var (key, value) in swaps)
+        {
+            mapping[key] = value;
+        }
+    }
+
+    private static void ResolveSequenceAliases(SequenceDataNode sequence, DocumentState state)
+    {
+        for (var i = 0; i < sequence.Count; i++)
+        {
+            if (sequence[i] is DataNodeAlias alias)
+                sequence[i] = ResolveAlias(alias, state);
+        }
+    }
+
+    private static DataNode ResolveAlias(DataNodeAlias alias, DocumentState state)
+    {
+        if (!state.Anchors.TryGetValue(alias.Anchor, out var node))
+            throw new DataParseException($"Unable to resolve alias '{alias.Anchor}'");
+
+        return node;
+    }
+
+    private static string ConvertTag(TagName tag)
+    {
+        return (tag.IsNonSpecific || tag.IsEmpty) ? null : tag.Value;
+    }
+
+    private sealed class DocumentState
+    {
+        public readonly Dictionary<AnchorName, DataNode> Anchors = new();
+        public ValueList<DataNode> UnresolvedAliasOwners;
+    }
+
+    private sealed class DataNodeAlias : DataNode
+    {
+        public readonly AnchorName Anchor;
+
+        public DataNodeAlias(AnchorName anchor) : base(default, default)
+        {
+            Anchor = anchor;
+        }
+
+        public override bool IsEmpty => true;
+
+        public override DataNode Copy()
+        {
+            throw new NotSupportedException();
+        }
+
+        public override DataNode Except(DataNode node)
+        {
+            throw new NotSupportedException();
+        }
+
+        public override DataNode PushInheritance(DataNode parent)
+        {
+            throw new NotSupportedException();
+        }
+    }
+}
+
+public sealed class DataParseException : Exception
+{
+    public DataParseException()
+    {
+    }
+
+    public DataParseException(string message) : base(message)
+    {
+    }
+
+    public DataParseException(string message, Exception inner) : base(message, inner)
+    {
+    }
+}
+
+public sealed record DataNodeDocument(DataNode Root);
+