Move upload commands to engine (#4072)

This commit is contained in:
Leon Friedrich
2023-05-20 13:53:05 +12:00
committed by GitHub
parent 36a124a3aa
commit ccec1cbdf3
17 changed files with 601 additions and 1 deletions

View File

@@ -0,0 +1,7 @@
uploadfolder-command-description = Uploads a folder from your UserData folder recursively to the server contentDB.
uploadfolder-command-help = uploadfolder [folder you want to upload in userdata/UploadFolder]
uploadfolder-command-wrong-args = Wrong number of arguments!
uploadfolder-command-folder-not-found = Folder {$folder} not found!
uploadfolder-command-resource-upload-disabled = Network Resource Uploading is currently disabled. check Server CVars.
uploadfolder-command-file-too-big = File {$filename} above the current size limit! It must be smaller than {$sizeLimit} MB. skipping.
uploadfolder-command-success = Uploaded {$fileCount} files

View File

@@ -20,6 +20,7 @@ using Robust.Client.ResourceManagement;
using Robust.Client.Serialization;
using Robust.Client.State;
using Robust.Client.Timing;
using Robust.Client.Upload;
using Robust.Client.UserInterface;
using Robust.Client.UserInterface.RichText;
using Robust.Client.UserInterface.Themes;
@@ -40,6 +41,7 @@ using Robust.Shared.Reflection;
using Robust.Shared.Replays;
using Robust.Shared.Serialization;
using Robust.Shared.Timing;
using Robust.Shared.Upload;
using Robust.Shared.ViewVariables;
namespace Robust.Client
@@ -90,6 +92,8 @@ namespace Robust.Client
deps.Register<IMidiManager, MidiManager>();
deps.Register<IAuthManager, AuthManager>();
deps.Register<ProfViewManager>();
deps.Register<IGamePrototypeLoadManager, GamePrototypeLoadManager>();
deps.Register<NetworkResourceManager>();
switch (mode)
{

View File

@@ -12,6 +12,7 @@ using Robust.Client.Input;
using Robust.Client.Placement;
using Robust.Client.ResourceManagement;
using Robust.Client.State;
using Robust.Client.Upload;
using Robust.Client.UserInterface;
using Robust.Client.UserInterface.RichText;
using Robust.Client.UserInterface.Themes;
@@ -34,6 +35,7 @@ using Robust.Shared.Serialization;
using Robust.Shared.Serialization.Manager;
using Robust.Shared.Threading;
using Robust.Shared.Timing;
using Robust.Shared.Upload;
using Robust.Shared.Utility;
using YamlDotNet.RepresentationModel;
@@ -75,6 +77,9 @@ namespace Robust.Client
[Dependency] private readonly IRuntimeLog _runtimeLog = default!;
[Dependency] private readonly ISerializationManager _serializationManager = default!;
[Dependency] private readonly MarkupTagManager _tagManager = default!;
[Dependency] private readonly IGamePrototypeLoadManager _protoLoadMan = default!;
[Dependency] private readonly NetworkResourceManager _netResMan = default!;
private IWebViewManagerHook? _webViewHook;
@@ -171,6 +176,8 @@ namespace Robust.Client
_client.Initialize();
_discord.Initialize();
_tagManager.Initialize();
_protoLoadMan.Initialize();
_netResMan.Initialize();
_userInterfaceManager.PostInitialize();
_modLoader.BroadcastRunLevel(ModRunLevel.PostInit);

View File

@@ -0,0 +1,34 @@
using System.IO;
using Robust.Client.UserInterface;
using Robust.Shared.Console;
using Robust.Shared.IoC;
using Robust.Shared.Upload;
namespace Robust.Client.Upload.Commands;
public sealed class LoadPrototypeCommand : IConsoleCommand
{
public string Command => "loadprototype";
public string Description => "Load a prototype file into the server.";
public string Help => Command;
public void Execute(IConsoleShell shell, string argStr, string[] args)
{
LoadPrototype();
}
public static async void LoadPrototype()
{
var dialogManager = IoCManager.Resolve<IFileDialogManager>();
var loadManager = IoCManager.Resolve<IGamePrototypeLoadManager>();
var stream = await dialogManager.OpenFile();
if (stream is null)
return;
// ew oop
var reader = new StreamReader(stream);
var proto = await reader.ReadToEndAsync();
loadManager.SendGamePrototype(proto);
}
}

View File

@@ -0,0 +1,65 @@
using Robust.Client.UserInterface;
using Robust.Shared;
using Robust.Shared.Configuration;
using Robust.Shared.Console;
using Robust.Shared.IoC;
using Robust.Shared.Network;
using Robust.Shared.Upload;
using Robust.Shared.Utility;
namespace Robust.Client.Upload.Commands;
public sealed class UploadFileCommand : IConsoleCommand
{
public string Command => "uploadfile";
public string Description => "Uploads a resource to the server.";
public string Help => $"{Command} [relative path for the resource]";
public async void Execute(IConsoleShell shell, string argStr, string[] args)
{
var cfgMan = IoCManager.Resolve<IConfigurationManager>();
if (!cfgMan.GetCVar(CVars.ResourceUploadingEnabled))
{
shell.WriteError("Network Resource Uploading is currently disabled by the server.");
return;
}
if (args.Length != 1)
{
shell.WriteError("Wrong number of arguments!");
return;
}
var path = new ResPath(args[0]).ToRelativePath();
var dialog = IoCManager.Resolve<IFileDialogManager>();
var filters = new FileDialogFilters(new FileDialogFilters.Group(path.Extension));
await using var file = await dialog.OpenFile(filters);
if (file == null)
{
shell.WriteError("Error picking file!");
return;
}
var sizeLimit = cfgMan.GetCVar(CVars.ResourceUploadingLimitMb);
if (sizeLimit > 0f && file.Length * SharedNetworkResourceManager.BytesToMegabytes > sizeLimit)
{
shell.WriteError($"File above the current size limit! It must be smaller than {sizeLimit} MB.");
return;
}
var data = file.CopyToArray();
var netManager = IoCManager.Resolve<INetManager>();
var msg = netManager.CreateNetMessage<NetworkResourceUploadMessage>();
msg.RelativePath = path;
msg.Data = data;
netManager.ClientSendMessage(msg);
}
}

View File

@@ -0,0 +1,78 @@
using System.IO;
using Robust.Shared;
using Robust.Shared.Configuration;
using Robust.Shared.Console;
using Robust.Shared.ContentPack;
using Robust.Shared.IoC;
using Robust.Shared.Localization;
using Robust.Shared.Network;
using Robust.Shared.Upload;
using Robust.Shared.Utility;
namespace Robust.Client.Upload.Commands;
public sealed class UploadFolderCommand : IConsoleCommand
{
public string Command => "uploadfolder";
public string Description => Loc.GetString("uploadfolder-command-description");
public string Help => Loc.GetString("uploadfolder-command-help");
private static readonly ResPath BaseUploadFolderPath = new("/UploadFolder");
[Dependency] private IResourceManager _resourceManager = default!;
[Dependency] private IConfigurationManager _configManager = default!;
public async void Execute(IConsoleShell shell, string argStr, string[] args)
{
var fileCount = 0;
if (!_configManager.GetCVar(CVars.ResourceUploadingEnabled))
{
shell.WriteError( Loc.GetString("uploadfolder-command-resource-upload-disabled"));
return;
}
if (args.Length != 1)
{
shell.WriteError( Loc.GetString("uploadfolder-command-wrong-args"));
shell.WriteLine( Loc.GetString("uploadfolder-command-help"));
return;
}
var folderPath = new ResPath(BaseUploadFolderPath + $"/{args[0]}");
if (!_resourceManager.UserData.Exists(folderPath.ToRootedPath()))
{
shell.WriteError( Loc.GetString("uploadfolder-command-folder-not-found",("folder", folderPath)));
return; // bomb out if the folder doesnt exist in /UploadFolder
}
//Grab all files in specified folder and upload them
foreach (var filepath in _resourceManager.UserData.Find($"{folderPath.ToRelativePath()}/").files )
{
await using var filestream = _resourceManager.UserData.Open(filepath,FileMode.Open);
{
var sizeLimit = _configManager.GetCVar(CVars.ResourceUploadingLimitMb);
if (sizeLimit > 0f && filestream.Length * SharedNetworkResourceManager.BytesToMegabytes > sizeLimit)
{
shell.WriteError( Loc.GetString("uploadfolder-command-file-too-big", ("filename",filepath), ("sizeLimit",sizeLimit)));
return;
}
var data = filestream.CopyToArray();
var netManager = IoCManager.Resolve<INetManager>();
var msg = netManager.CreateNetMessage<NetworkResourceUploadMessage>();
msg.RelativePath = new ResPath($"{filepath.ToString().Remove(0,14)}"); //removes /UploadFolder/ from path
msg.Data = data;
netManager.ClientSendMessage(msg);
fileCount++;
}
}
shell.WriteLine( Loc.GetString("uploadfolder-command-success",("fileCount",fileCount)));
}
}

View File

@@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using Robust.Shared.IoC;
using Robust.Shared.Localization;
using Robust.Shared.Log;
using Robust.Shared.Network;
using Robust.Shared.Prototypes;
using Robust.Shared.Upload;
namespace Robust.Client.Upload;
public sealed class GamePrototypeLoadManager : IGamePrototypeLoadManager
{
[Dependency] private readonly IClientNetManager _netManager = default!;
[Dependency] private readonly IPrototypeManager _prototypeManager = default!;
[Dependency] private readonly ILocalizationManager _localizationManager = default!;
public void Initialize()
{
_netManager.RegisterNetMessage<GamePrototypeLoadMessage>(LoadGamePrototype);
}
private void LoadGamePrototype(GamePrototypeLoadMessage message)
{
var changed = new Dictionary<Type, HashSet<string>>();
_prototypeManager.LoadString(message.PrototypeData, true, changed);
_prototypeManager.ResolveResults();
_prototypeManager.ReloadPrototypes(changed);
_localizationManager.ReloadLocalizations();
Logger.InfoS("adminbus", "Loaded adminbus prototype data.");
}
public void SendGamePrototype(string prototype)
{
var msg = new GamePrototypeLoadMessage
{
PrototypeData = prototype
};
_netManager.ClientSendMessage(msg);
}
}

View File

@@ -0,0 +1,15 @@
using Robust.Shared.Upload;
namespace Robust.Client.Upload;
public sealed class NetworkResourceManager : SharedNetworkResourceManager
{
/// <summary>
/// Callback for when the server sends a new resource.
/// </summary>
/// <param name="msg">The network message containing the data.</param>
protected override void ResourceUploadMsg(NetworkResourceUploadMessage msg)
{
ContentRoot.AddOrUpdateFile(msg.RelativePath, msg.Data);
}
}

View File

@@ -15,6 +15,7 @@ using Robust.Server.Replays;
using Robust.Server.Scripting;
using Robust.Server.ServerHub;
using Robust.Server.ServerStatus;
using Robust.Server.Upload;
using Robust.Server.Utility;
using Robust.Server.ViewVariables;
using Robust.Shared;
@@ -35,6 +36,7 @@ using Robust.Shared.Serialization;
using Robust.Shared.Serialization.Manager;
using Robust.Shared.Threading;
using Robust.Shared.Timing;
using Robust.Shared.Upload;
using Robust.Shared.Utility;
using Serilog.Debugging;
using Serilog.Sinks.Loki;
@@ -98,6 +100,8 @@ namespace Robust.Server
[Dependency] private readonly IStatusHost _statusHost = default!;
[Dependency] private readonly IComponentFactory _componentFactory = default!;
[Dependency] private readonly IInternalReplayRecordingManager _replay = default!;
[Dependency] private readonly IGamePrototypeLoadManager _protoLoadMan = default!;
[Dependency] private readonly NetworkResourceManager _netResMan = default!;
private readonly Stopwatch _uptimeStopwatch = new();
@@ -377,6 +381,8 @@ namespace Robust.Server
_stateManager.TransformNetId = reg.NetID.Value;
_scriptHost.Initialize();
_protoLoadMan.Initialize();
_netResMan.Initialize();
_modLoader.BroadcastRunLevel(ModRunLevel.PostInit);

View File

@@ -13,6 +13,7 @@ using Robust.Server.Scripting;
using Robust.Server.Serialization;
using Robust.Server.ServerHub;
using Robust.Server.ServerStatus;
using Robust.Server.Upload;
using Robust.Server.ViewVariables;
using Robust.Shared;
using Robust.Shared.Configuration;
@@ -22,13 +23,13 @@ using Robust.Shared.GameObjects;
using Robust.Shared.IoC;
using Robust.Shared.Map;
using Robust.Shared.Network;
using Robust.Shared.Physics;
using Robust.Shared.Players;
using Robust.Shared.Prototypes;
using Robust.Shared.Reflection;
using Robust.Shared.Replays;
using Robust.Shared.Serialization;
using Robust.Shared.Timing;
using Robust.Shared.Upload;
using Robust.Shared.ViewVariables;
namespace Robust.Server
@@ -90,6 +91,8 @@ namespace Robust.Server
deps.Register<IConfigurationManagerInternal, ServerNetConfigurationManager>();
deps.Register<IServerNetConfigurationManager, ServerNetConfigurationManager>();
deps.Register<INetConfigurationManagerInternal, ServerNetConfigurationManager>();
deps.Register<IGamePrototypeLoadManager, GamePrototypeLoadManager>();
deps.Register<NetworkResourceManager>();
}
}
}

View File

@@ -0,0 +1,96 @@
using System;
using System.Collections.Generic;
using Robust.Server.Console;
using Robust.Server.Player;
using Robust.Shared.IoC;
using Robust.Shared.Localization;
using Robust.Shared.Log;
using Robust.Shared.Network;
using Robust.Shared.Prototypes;
using Robust.Shared.Replays;
using Robust.Shared.Serialization.Markdown.Mapping;
using Robust.Shared.Upload;
namespace Robust.Server.Upload;
/// <summary>
/// Manages sending runtime-loaded prototypes from game staff to clients.
/// </summary>
public sealed class GamePrototypeLoadManager : IGamePrototypeLoadManager
{
[Dependency] private readonly IReplayRecordingManager _replay = default!;
[Dependency] private readonly IServerNetManager _netManager = default!;
[Dependency] private readonly IPlayerManager _playerManager = default!;
[Dependency] private readonly IPrototypeManager _prototypeManager = default!;
[Dependency] private readonly ILocalizationManager _localizationManager = default!;
[Dependency] private readonly IConGroupController _controller = default!;
private readonly List<string> _loadedPrototypes = new();
public IReadOnlyList<string> LoadedPrototypes => _loadedPrototypes;
public void Initialize()
{
_netManager.RegisterNetMessage<GamePrototypeLoadMessage>(ClientLoadsPrototype);
_netManager.Connected += NetManagerOnConnected;
_replay.OnRecordingStarted += OnStartReplayRecording;
}
private void OnStartReplayRecording((MappingDataNode, List<object>) initReplayData)
{
// replays will need information about currently loaded prototypes
foreach (var prototype in _loadedPrototypes)
{
initReplayData.Item2.Add(new ReplayPrototypeUploadMsg { PrototypeData = prototype });
}
}
public void SendGamePrototype(string prototype)
{
}
private void ClientLoadsPrototype(GamePrototypeLoadMessage message)
{
var player = _playerManager.GetSessionByChannel(message.MsgChannel);
if (_controller.CanCommand(player, "loadprototype"))
{
LoadPrototypeData(message.PrototypeData);
Logger.InfoS("adminbus", $"Loaded adminbus prototype data from {player.Name}.");
}
else
{
message.MsgChannel.Disconnect("Sent prototype message without permission!");
}
}
private void LoadPrototypeData(string prototypeData)
{
_loadedPrototypes.Add(prototypeData);
_replay.QueueReplayMessage(new ReplayPrototypeUploadMsg { PrototypeData = prototypeData });
var msg = new GamePrototypeLoadMessage
{
PrototypeData = prototypeData
};
_netManager.ServerSendToAll(msg); // everyone load it up!
var changed = new Dictionary<Type, HashSet<string>>();
_prototypeManager.LoadString(prototypeData, true, changed); // server needs it too.
_prototypeManager.ResolveResults();
_prototypeManager.ReloadPrototypes(changed);
_localizationManager.ReloadLocalizations();
}
private void NetManagerOnConnected(object? sender, NetChannelArgs e)
{
// Just dump all the prototypes on connect, before them missing could be an issue.
foreach (var prototype in _loadedPrototypes)
{
var msg = new GamePrototypeLoadMessage
{
PrototypeData = prototype
};
e.Channel.SendMessage(msg);
}
}
}

View File

@@ -0,0 +1,92 @@
using System;
using System.Collections.Generic;
using Robust.Server.Console;
using Robust.Server.Player;
using Robust.Shared;
using Robust.Shared.Configuration;
using Robust.Shared.IoC;
using Robust.Shared.Network;
using Robust.Shared.Replays;
using Robust.Shared.Serialization.Markdown.Mapping;
using Robust.Shared.Upload;
using Robust.Shared.ViewVariables;
namespace Robust.Server.Upload;
public sealed class NetworkResourceManager : SharedNetworkResourceManager
{
[Dependency] private readonly IPlayerManager _playerManager = default!;
[Dependency] private readonly IServerNetManager _serverNetManager = default!;
[Dependency] private readonly IConfigurationManager _cfgManager = default!;
[Dependency] private readonly IReplayRecordingManager _replay = default!;
[Dependency] private readonly IConGroupController _controller = default!;
public event Action<IPlayerSession, NetworkResourceUploadMessage>? OnResourceUploaded;
[ViewVariables] public bool Enabled { get; private set; } = true;
[ViewVariables] public float SizeLimit { get; private set; }
public override void Initialize()
{
base.Initialize();
_serverNetManager.Connected += ServerNetManagerOnConnected;
_cfgManager.OnValueChanged(CVars.ResourceUploadingEnabled, value => Enabled = value, true);
_cfgManager.OnValueChanged(CVars.ResourceUploadingLimitMb, value => SizeLimit = value, true);
_replay.OnRecordingStarted += OnStartReplayRecording;
}
private void OnStartReplayRecording((MappingDataNode, List<object>) initReplayData)
{
// replays will need information about currently loaded extra resources
foreach (var (path, data) in ContentRoot.GetAllFiles())
{
initReplayData.Item2.Add(new ReplayResourceUploadMsg { RelativePath = path, Data = data });
}
}
/// <summary>
/// Callback for when a client attempts to upload a resource.
/// </summary>
/// <param name="msg"></param>
/// <exception cref="NotImplementedException"></exception>
protected override void ResourceUploadMsg(NetworkResourceUploadMessage msg)
{
// Do not allow uploading any new resources if it has been disabled.
// Note: Any resources uploaded before being disabled will still be kept and sent.
if (!Enabled)
return;
if (!_playerManager.TryGetSessionByChannel(msg.MsgChannel, out var session))
return;
if (!_controller.CanCommand(session, "uploadfile"))
return;
// Ensure the data is under the current size limit, if it's currently enabled.
if (SizeLimit > 0f && msg.Data.Length * BytesToMegabytes > SizeLimit)
return;
ContentRoot.AddOrUpdateFile(msg.RelativePath, msg.Data);
// Now we broadcast the message!
foreach (var channel in _serverNetManager.Channels)
{
channel.SendMessage(msg);
}
_replay.QueueReplayMessage(new ReplayResourceUploadMsg { RelativePath = msg.RelativePath, Data = msg.Data });
OnResourceUploaded?.Invoke(session, msg);
}
private void ServerNetManagerOnConnected(object? sender, NetChannelArgs e)
{
foreach (var (path, data) in ContentRoot.GetAllFiles())
{
var msg = new NetworkResourceUploadMessage();
msg.RelativePath = path;
msg.Data = data;
e.Channel.SendMessage(msg);
}
}
}

View File

@@ -1453,5 +1453,23 @@ namespace Robust.Shared
/// and will complain if anything unknown is found (probably indicating a typo of some kind).
/// </remarks>
public static readonly CVarDef<bool> CfgCheckUnused = CVarDef.Create("cfg.check_unused", true);
/*
* Network Resource Manager
*/
/// <summary>
/// Controls whether new resources can be uploaded by admins.
/// Does not prevent already uploaded resources from being sent.
/// </summary>
public static readonly CVarDef<bool> ResourceUploadingEnabled =
CVarDef.Create("netres.enabled", true, CVar.REPLICATED | CVar.SERVER);
/// <summary>
/// Controls the data size limit in megabytes for uploaded resources. If they're too big, they will be dropped.
/// Set to zero or a negative value to disable limit.
/// </summary>
public static readonly CVarDef<float> ResourceUploadingLimitMb =
CVarDef.Create("netres.limit", 3f, CVar.REPLICATED | CVar.SERVER);
}
}

View File

@@ -0,0 +1,22 @@
using Lidgren.Network;
using Robust.Shared.Network;
using Robust.Shared.Serialization;
namespace Robust.Shared.Upload;
public sealed class GamePrototypeLoadMessage : NetMessage
{
public override MsgGroups MsgGroup => MsgGroups.String;
public string PrototypeData { get; set; } = string.Empty;
public override void ReadFromBuffer(NetIncomingMessage buffer, IRobustSerializer serializer)
{
PrototypeData = buffer.ReadString();
}
public override void WriteToBuffer(NetOutgoingMessage buffer, IRobustSerializer serializer)
{
buffer.Write(PrototypeData);
}
}

View File

@@ -0,0 +1,16 @@
using System;
using Robust.Shared.Serialization;
namespace Robust.Shared.Upload;
public interface IGamePrototypeLoadManager
{
public void Initialize();
public void SendGamePrototype(string prototype);
}
[Serializable, NetSerializable]
public sealed class ReplayPrototypeUploadMsg
{
public string PrototypeData = default!;
}

View File

@@ -0,0 +1,42 @@
using System;
using Lidgren.Network;
using Robust.Shared.Network;
using Robust.Shared.Serialization;
using Robust.Shared.Utility;
namespace Robust.Shared.Upload;
public sealed class NetworkResourceUploadMessage : NetMessage
{
public override NetDeliveryMethod DeliveryMethod => NetDeliveryMethod.ReliableUnordered;
public override MsgGroups MsgGroup => MsgGroups.Command;
public byte[] Data { get; set; } = Array.Empty<byte>();
public ResPath RelativePath { get; set; } = ResPath.Self;
public NetworkResourceUploadMessage()
{
}
public NetworkResourceUploadMessage(byte[] data, ResPath relativePath)
{
Data = data;
RelativePath = relativePath;
}
public override void ReadFromBuffer(NetIncomingMessage buffer, IRobustSerializer serializer)
{
var dataLength = buffer.ReadVariableInt32();
Data = buffer.ReadBytes(dataLength);
// What is the second argument here?
RelativePath = new ResPath(buffer.ReadString());
}
public override void WriteToBuffer(NetOutgoingMessage buffer, IRobustSerializer serializer)
{
buffer.WriteVariableInt32(Data.Length);
buffer.Write(Data);
buffer.Write(RelativePath.ToString());
buffer.Write(ResPath.Separator);
}
}

View File

@@ -0,0 +1,54 @@
using System;
using Robust.Shared.ContentPack;
using Robust.Shared.IoC;
using Robust.Shared.Network;
using Robust.Shared.Serialization;
using Robust.Shared.Utility;
namespace Robust.Shared.Upload;
/// <summary>
/// Manager that allows resources to be added at runtime by admins.
/// They will be sent to all clients automatically.
/// </summary>
public abstract class SharedNetworkResourceManager : IDisposable
{
[Dependency] private readonly INetManager _netManager = default!;
[Dependency] protected readonly IResourceManager ResourceManager = default!;
public const double BytesToMegabytes = 0.000001d;
/// <summary>
/// The prefix for any and all downloaded network resources.
/// </summary>
private static readonly ResPath Prefix = ResPath.Root / "Uploaded";
protected readonly MemoryContentRoot ContentRoot = new();
//public bool FileExists(ResPath path)
// => ContentRoot.FileExists(path);
public virtual void Initialize()
{
_netManager.RegisterNetMessage<NetworkResourceUploadMessage>(ResourceUploadMsg);
// Add our content root to the resource manager.
ResourceManager.AddRoot(Prefix, ContentRoot);
}
protected abstract void ResourceUploadMsg(NetworkResourceUploadMessage msg);
public void Dispose()
{
// This is called automatically when the IoCManager's dependency collection is cleared.
// MemoryContentRoot uses a ReaderWriterLockSlim, which we need to dispose of.
ContentRoot.Dispose();
}
[Serializable, NetSerializable]
public sealed class ReplayResourceUploadMsg
{
public byte[] Data = default!;
public ResPath RelativePath = default!;
}
}