space-wizards/RobustToolbox
1
0
Fork 0
mirror of https://github.com/space-wizards/RobustToolbox.git synced 2026-02-14 19:29:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

benchmark workflow & storing it in db (#2653)

Browse Source
This commit is contained in:
Paul Ritter
2022-03-29 21:43:32 +02:00
committed by GitHub
parent 075b5ec203
commit 2fb64480db
11 changed files with 401 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/benchmarks.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
name: Benchmarks
on:
schedule:
- cron: '0 5 * * *'
push:
tags:
- 'v*'
env:
ROBUST_BENCHMARKS_SQL_ADDRESS: ${{ secrets.BENCHMARKS_WRITE_ADDRESS }}
ROBUST_BENCHMARKS_SQL_PORT: ${{ secrets.BENCHMARKS_WRITE_PORT }}
ROBUST_BENCHMARKS_SQL_USER: ${{ secrets.BENCHMARKS_WRITE_USER }}
ROBUST_BENCHMARKS_SQL_PASSWORD: ${{ secrets.BENCHMARKS_WRITE_PASSWORD }}
ROBUST_BENCHMARKS_SQL_DATABASE: benchmarks
jobs:
benchmark:
name: Run Benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup .NET Core
uses: actions/setup-dotnet@v1
with:
dotnet-version: 6.0.x
- name: Install dependencies
run: dotnet restore
- name: Run benchmark
run: cd Robust.Benchmarks && dotnet run --filter '*'

53
Robust.Benchmarks/Configs/DefaultSQLConfig.cs Normal file
View File

@@ -0,0 +1,53 @@
using System.Collections.Generic;
using System.Globalization;
using BenchmarkDotNet.Analysers;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Filters;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Validators;
using Robust.Benchmarks.Exporters;
namespace Robust.Benchmarks.Configs;
public sealed class DefaultSQLConfig : IConfig
{
public static readonly IConfig Instance = new DefaultSQLConfig();
private DefaultSQLConfig(){}
public IEnumerable<IExporter> GetExporters()
{
yield return SQLExporter.Default;
}
public IEnumerable<IColumnProvider> GetColumnProviders() => DefaultConfig.Instance.GetColumnProviders();
public IEnumerable<ILogger> GetLoggers() => DefaultConfig.Instance.GetLoggers();
public IEnumerable<IDiagnoser> GetDiagnosers() => DefaultConfig.Instance.GetDiagnosers();
public IEnumerable<IAnalyser> GetAnalysers() => DefaultConfig.Instance.GetAnalysers();
public IEnumerable<Job> GetJobs() => DefaultConfig.Instance.GetJobs();
public IEnumerable<IValidator> GetValidators() => DefaultConfig.Instance.GetValidators();
public IEnumerable<HardwareCounter> GetHardwareCounters() => DefaultConfig.Instance.GetHardwareCounters();
public IEnumerable<IFilter> GetFilters() => DefaultConfig.Instance.GetFilters();
public IEnumerable<BenchmarkLogicalGroupRule> GetLogicalGroupRules() => DefaultConfig.Instance.GetLogicalGroupRules();
public IOrderer Orderer => DefaultConfig.Instance.Orderer!;
public SummaryStyle SummaryStyle => DefaultConfig.Instance.SummaryStyle;
public ConfigUnionRule UnionRule => DefaultConfig.Instance.UnionRule;
public string ArtifactsPath => DefaultConfig.Instance.ArtifactsPath;
public CultureInfo CultureInfo => DefaultConfig.Instance.CultureInfo!;
public ConfigOptions Options => DefaultConfig.Instance.Options;
}

143
Robust.Benchmarks/Exporters/SQLExporter.cs Normal file
View File

@@ -0,0 +1,143 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Design;
using Npgsql;
namespace Robust.Benchmarks.Exporters;
public sealed class SQLExporter : IExporter
{
public static readonly IExporter Default = new SQLExporter();
private SQLExporter(){}
public void ExportToLog(Summary summary, ILogger logger)
{
Export(summary, logger);
}
public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
{
Export(summary, consoleLogger);
return Array.Empty<string>();
}
private bool TryGetEnvironmentVariable(string name, ILogger logger, [NotNullWhen(true)] out string? value)
{
value = Environment.GetEnvironmentVariable("name");
if (value == null)
logger.WriteError($"ROBUST_BENCHMARKS_ENABLE_SQL is set, but {name} is missing.");
return value != null;
}
private void Export(Summary summary, ILogger logger)
{
if (!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_ADDRESS", logger, out var address) ||
!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_PORT", logger, out var rawPort) ||
!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_USER", logger, out var user) ||
!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_PASSWORD", logger, out var password) ||
!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_DATABASE", logger, out var db) ||
!TryGetEnvironmentVariable("GITHUB_SHA", logger, out var gitHash))
return;
if (!int.TryParse(rawPort, out var port))
{
logger.WriteError("Failed parsing ROBUST_BENCHMARKS_SQL_PORT to int.");
return;
}
var builder = new DbContextOptionsBuilder<BenchmarkContext>();
var connectionString = new NpgsqlConnectionStringBuilder
{
Host = address,
Port = port,
Database = db,
Username = user,
Password = password
}.ConnectionString;
builder.UseNpgsql(connectionString);
using var ctx = new BenchmarkContext(builder.Options);
try
{
ctx.Database.Migrate();
ctx.BenchmarkRuns.Add(BenchmarkRun.FromSummary(summary, gitHash));
ctx.SaveChanges();
}
finally
{
ctx.Dispose();
}
}
public string Name => "sql";
}
public sealed class DesignTimeContextFactoryPostgres : IDesignTimeDbContextFactory<BenchmarkContext>
{
public BenchmarkContext CreateDbContext(string[] args)
{
var optionsBuilder = new DbContextOptionsBuilder<BenchmarkContext>();
optionsBuilder.UseNpgsql("Server=localhost");
return new BenchmarkContext(optionsBuilder.Options);
}
}
public class BenchmarkContext : DbContext
{
public DbSet<BenchmarkRun> BenchmarkRuns { get; set; } = default!;
public BenchmarkContext() { }
public BenchmarkContext(DbContextOptions<BenchmarkContext> options) : base(options) { }
}
public class BenchmarkRun
{
[Key, DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public ulong Id { get; set; }
public string GitHash { get; set; } = string.Empty;
[Column(TypeName = "timestamptz")]
public DateTime RunDate { get; set; }
public string Name { get; set; } = string.Empty;
[Column(TypeName = "jsonb")]
public BenchmarkRunReport[] Reports { get; set; } = Array.Empty<BenchmarkRunReport>();
public static BenchmarkRun FromSummary(Summary summary, string gitHash)
{
return new BenchmarkRun
{
Reports = summary.Reports.Select(r => new BenchmarkRunReport
{
Parameters = r.BenchmarkCase.Parameters.Items.Select(p => new BenchmarkRunParameter
{
Name = p.Name,
Value = p.Value
}).ToArray(),
Statistics = r.ResultStatistics
}).ToArray(),
Name = summary.BenchmarksCases.First().FolderInfo,
RunDate = DateTime.UtcNow,
GitHash = gitHash
};
}
}
public class BenchmarkRunReport
{
public BenchmarkRunParameter[] Parameters { get; set; } = Array.Empty<BenchmarkRunParameter>();
public Statistics Statistics { get; set; } = default!;
}
public class BenchmarkRunParameter
{
public string Name { get; set; } = string.Empty;
public object Value { get; set; } = default!;
}

55
Robust.Benchmarks/Migrations/20220328231938_InitialCreate.Designer.cs generated Normal file
View File

@@ -0,0 +1,55 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using Robust.Benchmarks.Exporters;
#nullable disable
namespace Robust.Benchmarks.Migrations
{
[DbContext(typeof(BenchmarkContext))]
[Migration("20220328231938_InitialCreate")]
partial class InitialCreate
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "6.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Robust.Benchmarks.Exporters.BenchmarkRun", b =>
{
b.Property<decimal>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(20,0)");
b.Property<string>("GitHash")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<BenchmarkRunReport[]>("Reports")
.IsRequired()
.HasColumnType("jsonb");
b.Property<DateTime>("RunDate")
.HasColumnType("Date");
b.HasKey("Id");
b.ToTable("BenchmarkRuns");
});
#pragma warning restore 612, 618
}
}
}

35
Robust.Benchmarks/Migrations/20220328231938_InitialCreate.cs Normal file
View File

@@ -0,0 +1,35 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
using Robust.Benchmarks.Exporters;
#nullable disable
namespace Robust.Benchmarks.Migrations
{
public partial class InitialCreate : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "BenchmarkRuns",
columns: table => new
{
Id = table.Column<decimal>(type: "numeric(20,0)", nullable: false),
GitHash = table.Column<string>(type: "text", nullable: false),
RunDate = table.Column<DateTime>(type: "Date", nullable: false),
Name = table.Column<string>(type: "text", nullable: false),
Reports = table.Column<BenchmarkRunReport[]>(type: "jsonb", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_BenchmarkRuns", x => x.Id);
});
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "BenchmarkRuns");
}
}
}

53
Robust.Benchmarks/Migrations/BenchmarkContextModelSnapshot.cs Normal file
View File

@@ -0,0 +1,53 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
using Robust.Benchmarks.Exporters;
#nullable disable
namespace Robust.Benchmarks.Migrations
{
[DbContext(typeof(BenchmarkContext))]
partial class BenchmarkContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "6.0.0")
.HasAnnotation("Relational:MaxIdentifierLength", 63);
NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
modelBuilder.Entity("Robust.Benchmarks.Exporters.BenchmarkRun", b =>
{
b.Property<decimal>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("numeric(20,0)");
b.Property<string>("GitHash")
.IsRequired()
.HasColumnType("text");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("text");
b.Property<BenchmarkRunReport[]>("Reports")
.IsRequired()
.HasColumnType("jsonb");
b.Property<DateTime>("RunDate")
.HasColumnType("Date");
b.HasKey("Id");
b.ToTable("BenchmarkRuns");
});
#pragma warning restore 612, 618
}
}
}

3
Robust.Benchmarks/NumericsHelpers/AddBenchmark.cs
View File

@@ -9,6 +9,9 @@ namespace Robust.Benchmarks.NumericsHelpers
[Params(32, 128)]
public int N { get; set; }
[Params(1,2)]
public int T { get; set; }
private float[] _inputA = default!;
private float[] _inputB = default!;
private float[] _output = default!;

5
Robust.Benchmarks/Program.cs
View File

@@ -1,6 +1,8 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Running;
using System;
using Robust.Benchmarks.Configs;
using Robust.Benchmarks.Exporters;
namespace Robust.Benchmarks
{
@@ -16,7 +18,8 @@ namespace Robust.Benchmarks
Console.WriteLine("THE DEBUG BUILD IS ONLY GOOD FOR FIXING A CRASHING BENCHMARK\n");
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig());
#else
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
var config = Environment.GetEnvironmentVariable("ROBUST_BENCHMARKS_ENABLE_SQL") != null ? DefaultSQLConfig.Instance : null;
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config);
#endif
}
}

5
Robust.Benchmarks/Robust.Benchmarks.csproj
View File

@@ -14,6 +14,11 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.0-rc.2" />
</ItemGroup>
<Import Project="..\MSBuild\Robust.Engine.targets" />
</Project>

11
Robust.Benchmarks/add-migration.ps1 Normal file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env pwsh
param([String]$name)
if ($name -eq "")
{
Write-Error "must specify migration name"
exit
}
dotnet ef migrations add --context BenchmarkContext -o Migrations $name

8
Robust.Benchmarks/add-migration.sh Normal file
View File

@@ -0,0 +1,8 @@
#!/usr/bin/env bash
if [ -z "$1" ] ; then
echo "Must specify migration name"
exit 1
fi
dotnet ef migrations add --context BenchmarkContext -o Migrations "$1"