space-wizards/RobustToolbox
1
0
Fork 0
mirror of https://github.com/space-wizards/RobustToolbox.git synced 2026-02-14 19:29:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus metrics server.

Browse Source 
Thanks to Redline for starting work on this and helping me out on Discord.
This commit is contained in:
Pieter-Jan Briers
2020-06-08 00:11:43 +02:00
parent 15432bf8a7
commit dd0cb9f216
12 changed files with 226 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Lidgren.Network/Lidgren.Network.csproj
View File

@@ -6,5 +6,6 @@
<Configurations>Debug;Release</Configurations>
<Platforms>AnyCPU</Platforms>
<SonarQubeExclude>true</SonarQubeExclude>
<DefineConstants>$(DefineConstants);USE_RELEASE_STATISTICS</DefineConstants>
</PropertyGroup>
</Project>

34
Robust.Server/BaseServer.cs
View File

@@ -1,8 +1,8 @@
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Threading;
using Prometheus;
using Robust.Server.Console;
using Robust.Server.Interfaces;
using Robust.Server.Interfaces.Console;
@@ -30,12 +30,13 @@ using Robust.Shared.Utility;
using Robust.Shared.Interfaces.Log;
using Robust.Shared.Interfaces.Resources;
using Robust.Shared.Exceptions;
using Robust.Shared.Localization;
using Robust.Server.Interfaces.Debugging;
using Robust.Server.Scripting;
using Robust.Server.ServerStatus;
using Robust.Shared;
using Robust.Shared.Network.Messages;
using Robust.Server.DataMetrics;
using Stopwatch = Robust.Shared.Timing.Stopwatch;
namespace Robust.Server
{
@@ -44,6 +45,19 @@ namespace Robust.Server
/// </summary>
internal sealed class BaseServer : IBaseServerInternal
{
private static readonly Gauge ServerUpTime = Metrics.CreateGauge(
"robust_server_uptime",
"The real time the server main loop has been running.");
private static readonly Gauge ServerCurTime = Metrics.CreateGauge(
"robust_server_curtime",
"The IGameTiming.CurTime of the server.");
private static readonly Gauge ServerCurTick = Metrics.CreateGauge(
"robust_server_curtick",
"The IGameTiming.CurTick of the server.");
#pragma warning disable 649
[Dependency] private readonly IConfigurationManager _config;
[Dependency] private readonly IComponentManager _components;
@@ -62,8 +76,11 @@ namespace Robust.Server
[Dependency] private readonly IModLoader _modLoader;
[Dependency] private readonly IWatchdogApi _watchdogApi;
[Dependency] private readonly IScriptHost _scriptHost;
[Dependency] private readonly IMetricsManager _metricsManager;
#pragma warning restore 649
private readonly Stopwatch _uptimeStopwatch = new Stopwatch();
private CommandLineArgs _commandLineArgs;
private FileLogHandler fileLogHandler;
private IGameLoop _mainLoop;
@@ -169,6 +186,9 @@ namespace Robust.Server
LoadSettings();
// Load metrics really early so that we can profile startup times in the future maybe.
_metricsManager.Initialize();
var netMan = IoCManager.Resolve<IServerNetManager>();
try
{
@@ -285,8 +305,15 @@ namespace Robust.Server
};
}
_uptimeStopwatch.Start();
_mainLoop.Tick += (sender, args) => Update(args);
_mainLoop.Update += (sender, args) =>
{
ServerUpTime.Set(_uptimeStopwatch.Elapsed.TotalSeconds);
};
// set GameLoop.Running to false to return from this function.
_mainLoop.Run();
@@ -394,6 +421,9 @@ namespace Robust.Server
private void Update(FrameEventArgs frameEventArgs)
{
ServerCurTick.Set(_time.CurTick.Value);
ServerCurTime.Set(_time.CurTime.TotalSeconds);
UpdateTitle();
_systemConsole.Update();

80
Robust.Server/DataMetrics/MetricsManager.cs Normal file
View File

@@ -0,0 +1,80 @@
using System;
using Prometheus;
using Robust.Shared.Interfaces.Configuration;
using Robust.Shared.IoC;
using Robust.Shared.Log;
#nullable enable
namespace Robust.Server.DataMetrics
{
internal sealed class MetricsManager : IMetricsManager, IPostInjectInit, IDisposable
{
[Dependency] private readonly IConfigurationManager _configurationManager = default!;
private bool _initialized;
private MetricServer? _metricServer;
public void Initialize()
{
_initialized = true;
Reload();
}
void IPostInjectInit.PostInject()
{
_configurationManager.RegisterCVar("metrics.enabled", false, onValueChanged: _ => Reload());
_configurationManager.RegisterCVar("metrics.host", "localhost", onValueChanged: _ => Reload());
_configurationManager.RegisterCVar("metrics.port", 44880, onValueChanged: _ => Reload());
}
private async void Stop()
{
if (_metricServer == null)
{
return;
}
Logger.InfoS("metrics", "Shutting down metrics.");
await _metricServer.StopAsync();
_metricServer = null;
}
void IDisposable.Dispose()
{
Stop();
_initialized = false;
}
private void Reload()
{
if (!_initialized)
{
return;
}
Stop();
var enabled = _configurationManager.GetCVar<bool>("metrics.enabled");
if (!enabled)
{
return;
}
var host = _configurationManager.GetCVar<string>("metrics.host");
var port = _configurationManager.GetCVar<int>("metrics.port");
Logger.InfoS("metrics", "Prometheus metrics enabled, host: {1} port: {0}", port, host);
_metricServer = new MetricServer(host, port);
_metricServer.Start();
}
}
internal interface IMetricsManager
{
void Initialize();
}
}

11
Robust.Server/GameObjects/ServerEntityManager.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Prometheus;
using Robust.Server.GameObjects.Components;
using Robust.Server.GameObjects.Components.Container;
using Robust.Server.Interfaces.GameObjects;
@@ -27,6 +28,9 @@ namespace Robust.Server.GameObjects
/// </summary>
public sealed class ServerEntityManager : EntityManager, IServerEntityManagerInternal
{
private static readonly Gauge EntitiesCount = Metrics.CreateGauge(
"robust_entities_count",
"Amount of alive entities.");
private const float MinimumMotionForMovers = 1 / 128f;
@@ -922,5 +926,12 @@ namespace Robust.Server.GameObjects
set.Remove(entity);
}
}
public override void Update(float frameTime)
{
base.Update(frameTime);
EntitiesCount.Set(AllEntities.Count);
}
}
}

8
Robust.Server/Player/PlayerManager.cs
View File

@@ -2,12 +2,12 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Prometheus;
using Robust.Server.Interfaces;
using Robust.Server.Interfaces.Player;
using Robust.Shared.Enums;
using Robust.Shared.GameStates;
using Robust.Shared.Input;
using Robust.Shared.Interfaces.GameObjects;
using Robust.Shared.Interfaces.Map;
using Robust.Shared.Interfaces.Network;
using Robust.Shared.Interfaces.Reflection;
@@ -26,6 +26,9 @@ namespace Robust.Server.Player
/// </summary>
internal class PlayerManager : IPlayerManager
{
private static readonly Gauge PlayerCountMetric = Metrics
.CreateGauge("robust_player_count", "Number of players on the server.");
#pragma warning disable 649
[Dependency] private readonly IBaseServer _baseServer;
[Dependency] private readonly IGameTiming _timing;
@@ -324,6 +327,8 @@ namespace Robust.Server.Player
{
_sessionsLock.ExitWriteLock();
}
PlayerCountMetric.Set(PlayerCount);
}
private void OnPlayerStatusChanged(IPlayerSession session, SessionStatus oldStatus, SessionStatus newStatus)
@@ -353,6 +358,7 @@ namespace Robust.Server.Player
_sessionsLock.ExitWriteLock();
}
PlayerCountMetric.Set(PlayerCount);
Dirty();
}

1
Robust.Server/Robust.Server.csproj
View File

@@ -16,6 +16,7 @@
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Core" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets" Version="2.2.1" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="3.1.1" />
<PackageReference Include="prometheus-net" Version="3.5.0" />
<PackageReference Include="YamlDotNet" Version="8.1.0" />
</ItemGroup>
<ItemGroup>

2
Robust.Server/ServerIoC.cs
View File

@@ -1,4 +1,5 @@
using Robust.Server.Console;
using Robust.Server.DataMetrics;
using Robust.Server.Debugging;
using Robust.Server.GameObjects;
using Robust.Server.GameStates;
@@ -73,6 +74,7 @@ namespace Robust.Server
IoCManager.Register<IDebugDrawingManager, DebugDrawingManager>();
IoCManager.Register<IWatchdogApi, WatchdogApi>();
IoCManager.Register<IScriptHost, ScriptHost>();
IoCManager.Register<IMetricsManager, MetricsManager>();
}
}
}

47
Robust.Shared/Network/NetManager.cs
View File

@@ -9,6 +9,7 @@ using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Lidgren.Network;
using Prometheus;
using Robust.Shared.Configuration;
using Robust.Shared.Interfaces.Configuration;
using Robust.Shared.Interfaces.Network;
@@ -35,6 +36,31 @@ namespace Robust.Shared.Network
/// </summary>
public partial class NetManager : IClientNetManager, IServerNetManager, IDisposable
{
private static readonly Counter SentPacketsMetrics = Metrics.CreateCounter(
"robust_net_sent_packets",
"Number of packets sent since server startup.");
private static readonly Counter RecvPacketsMetrics = Metrics.CreateCounter(
"robust_net_recv_packets",
"Number of packets sent since server startup.");
private static readonly Counter SentMessagesMetrics = Metrics.CreateCounter(
"robust_net_sent_messages",
"Number of messages sent since server startup.");
private static readonly Counter RecvMessagesMetrics = Metrics.CreateCounter(
"robust_net_recv_messages",
"Number of messages sent since server startup.");
private static readonly Counter SentBytesMetrics = Metrics.CreateCounter(
"robust_net_sent_bytes",
"Number of bytes sent since server startup.");
private static readonly Counter RecvBytesMetrics = Metrics.CreateCounter(
"robust_net_recv_bytes",
"Number of bytes sent since server startup.");
private readonly Dictionary<Type, ProcessMessage> _callbacks = new Dictionary<Type, ProcessMessage>();
/// <summary>
@@ -293,6 +319,13 @@ namespace Robust.Shared.Network
public void ProcessPackets()
{
var sentMessages = 0L;
var recvMessages = 0L;
var sentBytes = 0L;
var recvBytes = 0L;
var sentPackets = 0L;
var recvPackets = 0L;
foreach (var peer in _netPeers)
{
NetIncomingMessage msg;
@@ -343,6 +376,13 @@ namespace Robust.Shared.Network
peer.Recycle(msg);
}
}
sentMessages += peer.Statistics.SentMessages;
recvMessages += peer.Statistics.ReceivedMessages;
sentBytes += peer.Statistics.SentBytes;
recvBytes += peer.Statistics.ReceivedBytes;
sentPackets += peer.Statistics.SentPackets;
recvPackets += peer.Statistics.ReceivedPackets;
}
if (_toCleanNetPeers.Count != 0)
@@ -352,6 +392,13 @@ namespace Robust.Shared.Network
_netPeers.Remove(peer);
}
}
SentMessagesMetrics.IncTo(sentMessages);
RecvMessagesMetrics.IncTo(recvMessages);
SentBytesMetrics.IncTo(sentBytes);
RecvBytesMetrics.IncTo(recvBytes);
SentPacketsMetrics.IncTo(sentPackets);
RecvPacketsMetrics.IncTo(recvPackets);
}
/// <inheritdoc />

1
Robust.Shared/Robust.Shared.csproj
View File

@@ -20,6 +20,7 @@
<PackageReference Include="nfluidsynth" Version="0.3.0" />
<PackageReference Include="NGettext" Version="0.6.5" />
<PackageReference Include="Pidgin" Version="2.2.0" />
<PackageReference Include="prometheus-net" Version="3.5.0" />
<PackageReference Include="SharpZipLib" Version="1.2.0" />
<PackageReference Condition="'$(TargetFramework)' == 'net472'" Include="System.Memory" Version="4.5.3" />
<PackageReference Include="YamlDotNet" Version="8.1.0" />

14
Robust.Shared/Timing/GameLoop.cs
View File

@@ -5,6 +5,7 @@ using Robust.Shared.Log;
using Robust.Shared.Exceptions;
using Robust.Shared.IoC;
using Robust.Shared.Maths;
using Prometheus;
namespace Robust.Shared.Timing
{
@@ -48,6 +49,14 @@ namespace Robust.Shared.Timing
/// </summary>
public class GameLoop : IGameLoop
{
private static readonly Histogram _frameTimeHistogram = Metrics.CreateHistogram(
"robust_game_loop_frametime",
"Histogram of frametimes in ms",
new HistogramConfiguration
{
Buckets = Histogram.ExponentialBuckets(.0003, 1.5, 10)
});
private readonly IGameTiming _timing;
private TimeSpan _lastTick; // last wall time tick
private TimeSpan _lastKeepUp; // last wall time keep up announcement
@@ -177,7 +186,10 @@ namespace Robust.Shared.Timing
try
{
#endif
Tick?.Invoke(this, simFrameEvent);
using (_frameTimeHistogram.NewTimer())
{
Tick?.Invoke(this, simFrameEvent);
}
#if EXCEPTION_TOLERANCE
}
catch (Exception exp)

19
Tools/Docker/prometheus-grafana/docker-compose.yml Normal file
View File

@@ -0,0 +1,19 @@
version: "3.4"
services:
prometheus:
image: prom/prometheus
network_mode: host
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
grafana:
image: grafana/grafana
network_mode: host
volumes:
- grafana_data:/var/lib/grafana
- grafana_provisioning:/etc/grafana/provisioning/
volumes:
grafana_data:
grafana_provisioning:

12
Tools/Docker/prometheus-grafana/prometheus.yml Normal file
View File

@@ -0,0 +1,12 @@
global:
scrape_interval: 1s
evaluation_interval: 1s
rule_files:
# - "first.rules"
# - "second.rules"
scrape_configs:
- job_name: "robust"
static_configs:
- targets: ["localhost:1234"]