Serialization v3 aka constant suffering (#1606)

* oops

* fixes serialization il

* copytest

* typo & misc fixes

* 139 moment

* boxing

* mesa dum

* stuff

* goodbye bad friend

* last commit before the big (4) rewrite

* adds datanodes

* kills yamlobjserializer in favor of the new system

* adds more serializers, actually implements them & removes most of the last of the old system

* changed yamlfieldattribute namespace

* adds back iselfserialize

* refactors consts&flags

* renames everything to data(field/definition)

* adds afterserialization

* help

* dataclassgen

* fuggen help me mannen

* Fix most errors on content

* Fix engine errors except map loader

* maploader & misc fix

* misc fixes

* thing

* help

* refactors datanodes

* help me mannen

* Separate ITypeSerializer into reader and writer

* Convert all type serializers

* priority

* adds alot

* il fixes

* adds robustgen

* argh

* adds array & enum serialization

* fixes dataclasses

* adds vec2i / misc fixes

* fixes inheritance

* a very notcursed todo

* fixes some custom dataclasses

* push dis

* Remove data classes

* boutta box

* yes

* Add angle and regex serializer tests

* Make TypeSerializerTest abstract

* sets up ioc etc

* remove pushinheritance

* fixes

* Merge fixes, fix yaml hot reloading

* General fixes2

* Make enum serialization ignore case

* Fix the tag not being copied in data nodes

* Fix not properly serializing flag enums

* Fix component serialization on startup

* Implement ValueDataNode ToString

* Serialization IL fixes, fix return and string equality

* Remove async from prototype manager

* Make serializing unsupported node as enum exception more descriptive

* Fix serv3 tryread casting to serializer instead of reader

* Add constructor for invalid node type exception

* Temporary fix for SERV3: Turn populate delegate into regular code

* Fix not copying the data of non primitive types

* Fix not using the data definition found in copying

* Make ISerializationHooks require explicit implementations

* Add test for serialization inheritance

* Improve IsOverridenIn method

* Fix error message when a data definition is null

* Add method to cast a read value in Serv3Manager

* Rename IServ3Manager to ISerializationManager

* Rename usages of serv3manager, add generic copy method

* Fix IL copy method lookup

* Rename old usages of serv3manager

* Add ITypeCopier

* resistance is futile

* we will conquer this codebase

* Add copy method to all serializers

* Make primitive mismatch error message more descriptive

* bing bong im going to freacking heck

* oopsie moment

* hello are you interested in my wares

* does generic serializers under new architecture

* Convert every non generic serializer to the new format, general fixes

* Update usgaes of generic serializers, cleanup

* does some pushinheritance logic

* finishes pushinheritance FRAMEWORK

* shed

* Add box2, color and component registry serializer tests

* Create more deserialized types and store prototypes with their deserialized results

* Fixes and serializer updates

* Add serialization manager extensions

* adds pushinheritance

* Update all prototypes to have a parent and have consistent id/parent properties

* Fix grammar component serialization

* Add generic serializer tests

* thonk

* Add array serializer test

* Replace logger warning calls with exceptions

* fixes

* Move redundant methods to serialization manager extensions, cleanup

* Add array serialization

* fixes context

* more fixes

* argh

* inheritance

* this should do it

* fixes

* adds copiers & fixes some stuff

* copiers use context v1

* finishing copy context

* more context fixes

* Test fixes

* funky maps

* Fix server user interface component serialization

* Fix value tuple serialization

* Add copying for value types and arrays. Fix copy internal for primitives, enums and strings

* fixes

* fixes more stuff

* yes

* Make abstract/interface skips debugs instead of warnings

* Fix typo

* Make some dictionaries readonly

* Add checks for the serialization manager initializing and already being initialized

* Add base type required and usage for MeansDataDefinition and ImplicitDataDefinitionForInheritorsAttribute

* copy by ref

* Fix exception wording

* Update data field required summary with the new forbidden docs

* Use extension in map loader

* wanna erp

* Change serializing to not use il temporarily

* Make writing work with nullable types

* pushing

* check

* cuddling slaps HARD

* Add serialization priority test

* important fix

* a serialization thing

* serializer moment

* Add validation for some type serializers

* adds context

* moar context

* fixes

* Do the thing for appearance

* yoo lmao

* push haha pp

* Temporarily make copy delegate regular c# code

* Create deserialized component registry to handle not inheriting conflicting references

* YAML LINTER BABY

* ayes

* Fix sprite component norot not being default true like in latest master

* Remove redundant todos

* Add summary doc to every ISerializationManager method

* icon fixes

* Add skip hook argument to readers and copiers

* Merge fixes

* Fix ordering of arguments in read and copy reflection call

* Fix user interface components deserialization

* pew pew

* i am going to HECK

* Add MustUseReturnValue to copy-over methods

* Make serialization log calls use the same sawmill

* gamin

* Fix doc errors in ISerializationManager.cs

* goodbye brave soldier

* fixes

* WIP merge fixes and entity serialization

* aaaaaaaaaaaaaaa

* aaaaaaaaaaaaaaa

* adds inheritancebehaviour

* test/datafield fixes

* forgot that one

* adds more verbose validation

* This fixes the YAML hot reloading

* Replace yield break with Enumerable.Empty

* adds copiers

* aaaaaaaaaaaaa

* array fix
priority fix
misc fixes

* fix(?)

* fix.

* funny map serialization (wip)

* funny map serialization (wip)

* Add TODO

* adds proper info the validation

* Make yaml linter 5 times faster (~80% less execution time)

* Improves the error message for missing fields in the linter

* Include component name in unknown component type error node

* adds alwaysrelevant usa

* fixes mapsaving

* moved surpressor to analyzers proj

* warning cleanup & moves surpressor

* removes old msbuild targets

* Revert "Make yaml linter 5 times faster (~80% less execution time)"

This reverts commit 2ee4cc2c26.

* Add serialization to RobustServerSimulation and mock reflection methods
Fixes container tests

* Fix nullability warnings

* Improve yaml linter message feedback

* oops moment

* Add IEquatable, IComparable, ToString and operators to DataPosition
Rename it to NodeMark
Make it a readonly struct

* Remove try catch from enum parsing

* Make dependency management in serialization less bad

* Make dependencies an argument instead of a property on the serialization manager

* Clean up type serializers

* Improve validation messages and resourc epath checking

* Fix sprite error message

* reached perfection

Co-authored-by: Paul <ritter.paul1+git@googlemail.com>
Co-authored-by: DrSmugleaf <DrSmugleaf@users.noreply.github.com>
Co-authored-by: Vera Aguilera Puerto <zddm@outlook.es>
This commit is contained in:
Paul Ritter
2021-03-05 00:59:14 +01:00
committed by GitHub
parent 93018c9843
commit 80f9f24243
202 changed files with 8614 additions and 5345 deletions

View File

@@ -1,21 +1,26 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Linq;
using JetBrains.Annotations;
using Robust.Server.GameObjects;
using Robust.Shared.ContentPack;
using Robust.Shared.GameObjects;
using Robust.Shared.IoC;
using Robust.Shared.Log;
using Robust.Shared.Map;
using YamlDotNet.RepresentationModel;
using Robust.Shared.Utility;
using Robust.Shared.Serialization;
using Robust.Shared.GameObjects;
using System.Globalization;
using System.Linq;
using Robust.Server.GameObjects;
using Robust.Shared.ContentPack;
using Robust.Shared.Prototypes;
using Robust.Shared.Serialization;
using Robust.Shared.Serialization.Manager;
using Robust.Shared.Serialization.Manager.Result;
using Robust.Shared.Serialization.Markdown;
using Robust.Shared.Serialization.Markdown.Validation;
using Robust.Shared.Serialization.TypeSerializers.Interfaces;
using Robust.Shared.Timing;
using Robust.Shared.Utility;
using YamlDotNet.Core;
using YamlDotNet.RepresentationModel;
namespace Robust.Server.Maps
{
@@ -223,7 +228,10 @@ namespace Robust.Server.Maps
/// <summary>
/// Handles the primary bulk of state during the map serialization process.
/// </summary>
private class MapContext : YamlObjectSerializer.Context, IEntityLoadContext
private class MapContext : ISerializationContext, IEntityLoadContext,
ITypeSerializer<GridId, ValueDataNode>,
ITypeSerializer<EntityUid, ValueDataNode>,
ITypeReaderWriter<IEntity, ValueDataNode>
{
private readonly IMapManagerInternal _mapManager;
private readonly ITileDefinitionManager _tileDefinitionManager;
@@ -255,6 +263,10 @@ namespace Robust.Server.Maps
private Dictionary<ushort, string>? _tileMap;
public Dictionary<(Type, Type), object> TypeReaders { get; }
public Dictionary<Type, object> TypeWriters { get; }
public Dictionary<Type, object> TypeCopiers => TypeWriters;
public bool MapIsPostInit { get; private set; }
public MapContext(IMapManagerInternal maps, ITileDefinitionManager tileDefs,
@@ -269,6 +281,18 @@ namespace Robust.Server.Maps
_prototypeManager = prototypeManager;
RootNode = new YamlMappingNode();
TypeWriters = new Dictionary<Type, object>()
{
{typeof(IEntity), this},
{typeof(GridId), this},
{typeof(EntityUid), this}
};
TypeReaders = new Dictionary<(Type, Type), object>()
{
{(typeof(IEntity), typeof(ValueDataNode)), this},
{(typeof(GridId), typeof(ValueDataNode)), this},
{(typeof(EntityUid), typeof(ValueDataNode)), this}
};
}
public MapContext(IMapManagerInternal maps, ITileDefinitionManager tileDefs,
@@ -286,6 +310,18 @@ namespace Robust.Server.Maps
RootNode = node;
TargetMap = targetMapId;
_prototypeManager = prototypeManager;
TypeWriters = new Dictionary<Type, object>()
{
{typeof(IEntity), this},
{typeof(GridId), this},
{typeof(EntityUid), this}
};
TypeReaders = new Dictionary<(Type, Type), object>()
{
{(typeof(IEntity), typeof(ValueDataNode)), this},
{(typeof(GridId), typeof(ValueDataNode)), this},
{(typeof(EntityUid), typeof(ValueDataNode)), this}
};
}
// Deserialization
@@ -530,7 +566,10 @@ namespace Robust.Server.Maps
{
foreach (var compData in componentList)
{
CurrentReadingEntityComponents[compData["type"].AsString()] = (YamlMappingNode) compData;
var copy = new YamlMappingNode(((YamlMappingNode)compData).AsEnumerable());
copy.Children.Remove(new YamlScalarNode("type"));
//TODO Paul: maybe replace mapping with datanode
CurrentReadingEntityComponents[compData["type"].AsString()] = copy;
}
}
@@ -687,9 +726,11 @@ namespace Robust.Server.Maps
private void WriteEntitySection()
{
var serializationManager = IoCManager.Resolve<ISerializationManager>();
var entities = new YamlSequenceNode();
RootNode.Add("entities", entities);
var prototypeCompCache = new Dictionary<string, Dictionary<string, MappingDataNode>>();
foreach (var entity in Entities.OrderBy(e => EntityUidMap[e.Uid]))
{
CurrentWritingEntity = entity;
@@ -701,6 +742,14 @@ namespace Robust.Server.Maps
if (entity.Prototype != null)
{
mapping.Add("type", entity.Prototype.ID);
if (!prototypeCompCache.ContainsKey(entity.Prototype.ID))
{
prototypeCompCache[entity.Prototype.ID] = new Dictionary<string, MappingDataNode>();
foreach (var (compType, comp) in entity.Prototype.Components)
{
prototypeCompCache[entity.Prototype.ID].Add(compType, serializationManager.WriteValueAs<MappingDataNode>(comp.GetType(), comp));
}
}
}
var components = new YamlSequenceNode();
@@ -710,18 +759,21 @@ namespace Robust.Server.Maps
if (component is MapSaveIdComponent)
continue;
var compMapping = new YamlMappingNode();
CurrentWritingComponent = component.Name;
var compSerializer = YamlObjectSerializer.NewWriter(compMapping, this);
var compMapping = serializationManager.WriteValueAs<MappingDataNode>(component.GetType(), component, context: this);
component.ExposeData(compSerializer);
if (entity.Prototype != null && prototypeCompCache[entity.Prototype.ID].TryGetValue(component.Name, out var protMapping))
{
compMapping = compMapping.Except(protMapping);
if(compMapping == null) continue;
}
// Don't need to write it if nothing was written!
if (compMapping.Children.Count != 0)
{
compMapping.AddNode("type", new ValueDataNode(component.Name));
// Something actually got written!
compMapping.Add("type", component.Name);
components.Add(compMapping);
components.Add(compMapping.ToYamlNode());
}
}
@@ -734,140 +786,32 @@ namespace Robust.Server.Maps
}
}
public override bool TryNodeToType(YamlNode node, Type type, [NotNullWhen(true)] out object? obj)
{
if (type == typeof(GridId))
{
if (node.AsString() == "null")
{
obj = GridId.Invalid;
return true;
}
var val = node.AsInt();
if (val >= Grids.Count)
{
Logger.ErrorS("map", "Error in map file: found local grid ID '{0}' which does not exist.", val);
}
else
{
obj = Grids[val].Index;
return true;
}
}
if (type == typeof(EntityUid))
{
if (node.AsString() == "null")
{
obj = EntityUid.Invalid;
return true;
}
var val = node.AsInt();
if (val >= Entities.Count)
{
Logger.ErrorS("map", "Error in map file: found local entity UID '{0}' which does not exist.",
val);
}
else
{
obj = UidEntityMap[val];
return true;
}
}
if (typeof(IEntity).IsAssignableFrom(type))
{
var val = node.AsInt();
if (val >= Entities.Count)
{
Logger.ErrorS("map", "Error in map file: found local entity UID '{0}' which does not exist.",
val);
}
else
{
obj = Entities[val];
return true;
}
}
obj = null;
return false;
}
public override bool TryTypeToNode(object obj, [NotNullWhen(true)] out YamlNode? node)
{
switch (obj)
{
case GridId gridId:
if (!GridIDMap.TryGetValue(gridId, out var gridMapped))
{
Logger.WarningS("map", "Cannot write grid ID '{0}', falling back to nullspace.", gridId);
break;
}
else
{
node = new YamlScalarNode(gridMapped.ToString(CultureInfo.InvariantCulture));
return true;
}
case EntityUid entityUid:
if (!EntityUidMap.TryGetValue(entityUid, out var entityUidMapped))
{
// Terrible hack to mute this warning on the grids themselves when serializing blueprints.
if (!IsBlueprintMode || !CurrentWritingEntity!.HasComponent<MapGridComponent>() ||
CurrentWritingComponent != "Transform")
{
Logger.WarningS("map", "Cannot write entity UID '{0}'.", entityUid);
}
node = new YamlScalarNode("null");
return true;
}
else
{
node = new YamlScalarNode(entityUidMapped.ToString(CultureInfo.InvariantCulture));
return true;
}
case IEntity entity:
if (!EntityUidMap.TryGetValue(entity.Uid, out var entityMapped))
{
Logger.WarningS("map", "Cannot write entity UID '{0}'.", entity.Uid);
break;
}
else
{
node = new YamlScalarNode(entityMapped.ToString(CultureInfo.InvariantCulture));
return true;
}
}
node = null;
return false;
}
// Create custom object serializers that will correctly allow data to be overriden by the map file.
ObjectSerializer IEntityLoadContext.GetComponentSerializer(string componentName, YamlMappingNode? protoData)
IComponent IEntityLoadContext.GetComponentData(string componentName,
IComponent? protoData)
{
if (CurrentReadingEntityComponents == null)
{
throw new InvalidOperationException();
}
var list = new List<YamlMappingNode>();
var serializationManager = IoCManager.Resolve<ISerializationManager>();
var factory = IoCManager.Resolve<IComponentFactory>();
IComponent data = protoData != null
? serializationManager.CreateCopy(protoData, this)!
: (IComponent) Activator.CreateInstance(factory.GetRegistration(componentName).Type)!;
if (CurrentReadingEntityComponents.TryGetValue(componentName, out var mapping))
{
list.Add(mapping);
var mapData = (IDeserializedDefinition) serializationManager.Read(
factory.GetRegistration(componentName).Type,
mapping.ToDataNode(), this);
var newData = serializationManager.PopulateDataDefinition(data, mapData);
data = (IComponent) newData.RawValue!;
}
if (protoData != null)
{
list.Add(protoData);
}
return YamlObjectSerializer.NewReader(list, this);
return data;
}
public IEnumerable<string> GetExtraComponentTypes()
@@ -875,34 +819,6 @@ namespace Robust.Server.Maps
return CurrentReadingEntityComponents!.Keys;
}
public override bool IsValueDefault<T>(string field, T value, WithFormat<T> format)
{
if (CurrentWritingEntity!.Prototype == null)
{
// No prototype, can't be default.
return false;
}
if (!CurrentWritingEntity.Prototype.Components.TryGetValue(CurrentWritingComponent!, out var compData))
{
// This component was added mid-game.
return false;
}
var testSer = YamlObjectSerializer.NewReader(compData);
if (testSer.TryReadDataFieldCached(field, format, out var prototypeVal))
{
if (value == null)
{
return prototypeVal == null;
}
return YamlObjectSerializer.IsSerializedEqual(value, prototypeVal);
}
return false;
}
private bool IsMapSavable(IEntity entity)
{
if (entity.Prototype?.MapSavable == false || !GridIDMap.ContainsKey(entity.Transform.GridID))
@@ -925,6 +841,173 @@ namespace Robust.Server.Maps
return true;
}
public DeserializationResult Read(ISerializationManager serializationManager, ValueDataNode node,
IDependencyCollection dependencies,
bool skipHook,
ISerializationContext? context = null)
{
if (node.Value == "null") return new DeserializedValue<GridId>(GridId.Invalid);
var val = int.Parse(node.Value);
if (val >= Grids.Count)
{
Logger.ErrorS("map", "Error in map file: found local grid ID '{0}' which does not exist.", val);
}
else
{
return new DeserializedValue<GridId>(Grids[val].Index);
}
return new DeserializedValue<GridId>(GridId.Invalid);
}
ValidationNode ITypeReader<IEntity, ValueDataNode>.Validate(ISerializationManager serializationManager,
ValueDataNode node, IDependencyCollection dependencies, ISerializationContext? context)
{
if (!int.TryParse(node.Value, out var val) || val >= Entities.Count)
{
return new ErrorNode(node, "Invalid EntityUid", true);
}
return new ValidatedValueNode(node);
}
ValidationNode ITypeReader<EntityUid, ValueDataNode>.Validate(ISerializationManager serializationManager,
ValueDataNode node, IDependencyCollection dependencies, ISerializationContext? context)
{
if (node.Value == "null")
{
return new ValidatedValueNode(node);
}
if (!int.TryParse(node.Value, out var val) || val >= Entities.Count)
{
return new ErrorNode(node, "Invalid EntityUid", true);
}
return new ValidatedValueNode(node);
}
ValidationNode ITypeReader<GridId, ValueDataNode>.Validate(ISerializationManager serializationManager,
ValueDataNode node, IDependencyCollection dependencies, ISerializationContext? context)
{
if (node.Value == "null") return new ValidatedValueNode(node);
if (!int.TryParse(node.Value, out var val) || val >= Grids.Count)
{
return new ErrorNode(node, "Invalid GridId", true);
}
return new ValidatedValueNode(node);
}
public DataNode Write(ISerializationManager serializationManager, IEntity value, bool alwaysWrite = false,
ISerializationContext? context = null)
{
if (!EntityUidMap.TryGetValue(value.Uid, out var entityMapped))
{
Logger.WarningS("map", "Cannot write entity UID '{0}'.", value.Uid);
return new ValueDataNode("");
}
else
{
return new ValueDataNode(entityMapped.ToString(CultureInfo.InvariantCulture));
}
}
public DataNode Write(ISerializationManager serializationManager, EntityUid value, bool alwaysWrite = false,
ISerializationContext? context = null)
{
if (!EntityUidMap.TryGetValue(value, out var entityUidMapped))
{
// Terrible hack to mute this warning on the grids themselves when serializing blueprints.
if (!IsBlueprintMode || !CurrentWritingEntity!.HasComponent<MapGridComponent>() ||
CurrentWritingComponent != "Transform")
{
Logger.WarningS("map", "Cannot write entity UID '{0}'.", value);
}
return new ValueDataNode("null");
}
else
{
return new ValueDataNode(entityUidMapped.ToString(CultureInfo.InvariantCulture));
}
}
public DataNode Write(ISerializationManager serializationManager, GridId value, bool alwaysWrite = false,
ISerializationContext? context = null)
{
if (!GridIDMap.TryGetValue(value, out var gridMapped))
{
Logger.WarningS("map", "Cannot write grid ID '{0}', falling back to nullspace.", gridMapped);
return new ValueDataNode("");
}
else
{
return new ValueDataNode(gridMapped.ToString(CultureInfo.InvariantCulture));
}
}
DeserializationResult ITypeReader<EntityUid, ValueDataNode>.Read(ISerializationManager serializationManager,
ValueDataNode node,
IDependencyCollection dependencies,
bool skipHook,
ISerializationContext? context)
{
if (node.Value == "null")
{
return new DeserializedValue<EntityUid>(EntityUid.Invalid);
}
var val = int.Parse(node.Value);
if (val >= Entities.Count)
{
Logger.ErrorS("map", "Error in map file: found local entity UID '{0}' which does not exist.", val);
}
else
{
return new DeserializedValue<EntityUid>(UidEntityMap[val]);
}
return new DeserializedValue<EntityUid>(EntityUid.Invalid);
}
DeserializationResult ITypeReader<IEntity, ValueDataNode>.Read(ISerializationManager serializationManager,
ValueDataNode node,
IDependencyCollection dependencies,
bool skipHook,
ISerializationContext? context)
{
var val = int.Parse(node.Value);
if (val >= Entities.Count)
{
Logger.ErrorS("map", "Error in map file: found local entity UID '{0}' which does not exist.", val);
return null!;
}
else
{
return new DeserializedValue<IEntity>(Entities[val]);
}
}
[MustUseReturnValue]
public GridId Copy(ISerializationManager serializationManager, GridId source, GridId target,
bool skipHook,
ISerializationContext? context = null)
{
return new(source.Value);
}
[MustUseReturnValue]
public EntityUid Copy(ISerializationManager serializationManager, EntityUid source, EntityUid target,
bool skipHook,
ISerializationContext? context = null)
{
return new((int) source);
}
}
/// <summary>