diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..d8d11e048 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,31 @@ +name: Benchmarks +on: + schedule: + - cron: '0 5 * * *' + push: + tags: + - 'v*' + +env: + ROBUST_BENCHMARKS_SQL_ADDRESS: ${{ secrets.BENCHMARKS_WRITE_ADDRESS }} + ROBUST_BENCHMARKS_SQL_PORT: ${{ secrets.BENCHMARKS_WRITE_PORT }} + ROBUST_BENCHMARKS_SQL_USER: ${{ secrets.BENCHMARKS_WRITE_USER }} + ROBUST_BENCHMARKS_SQL_PASSWORD: ${{ secrets.BENCHMARKS_WRITE_PASSWORD }} + ROBUST_BENCHMARKS_SQL_DATABASE: benchmarks + +jobs: + benchmark: + name: Run Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 6.0.x + - name: Install dependencies + run: dotnet restore + - name: Run benchmark + run: cd Robust.Benchmarks && dotnet run --filter '*' diff --git a/Robust.Benchmarks/Configs/DefaultSQLConfig.cs b/Robust.Benchmarks/Configs/DefaultSQLConfig.cs new file mode 100644 index 000000000..3dcd0f346 --- /dev/null +++ b/Robust.Benchmarks/Configs/DefaultSQLConfig.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Globalization; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Order; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Validators; +using Robust.Benchmarks.Exporters; + +namespace Robust.Benchmarks.Configs; + +public sealed class DefaultSQLConfig : IConfig +{ + public static readonly IConfig Instance = new DefaultSQLConfig(); + + private DefaultSQLConfig(){} + + public IEnumerable GetExporters() + { + yield return SQLExporter.Default; + } + + public IEnumerable GetColumnProviders() => DefaultConfig.Instance.GetColumnProviders(); + + public IEnumerable GetLoggers() => DefaultConfig.Instance.GetLoggers(); + + public IEnumerable GetDiagnosers() => DefaultConfig.Instance.GetDiagnosers(); + + public IEnumerable GetAnalysers() => DefaultConfig.Instance.GetAnalysers(); + + public IEnumerable GetJobs() => DefaultConfig.Instance.GetJobs(); + + public IEnumerable GetValidators() => DefaultConfig.Instance.GetValidators(); + + public IEnumerable GetHardwareCounters() => DefaultConfig.Instance.GetHardwareCounters(); + + public IEnumerable GetFilters() => DefaultConfig.Instance.GetFilters(); + + public IEnumerable GetLogicalGroupRules() => DefaultConfig.Instance.GetLogicalGroupRules(); + + public IOrderer Orderer => DefaultConfig.Instance.Orderer!; + public SummaryStyle SummaryStyle => DefaultConfig.Instance.SummaryStyle; + public ConfigUnionRule UnionRule => DefaultConfig.Instance.UnionRule; + public string ArtifactsPath => DefaultConfig.Instance.ArtifactsPath; + public CultureInfo CultureInfo => DefaultConfig.Instance.CultureInfo!; + public ConfigOptions Options => DefaultConfig.Instance.Options; +} diff --git a/Robust.Benchmarks/Exporters/SQLExporter.cs b/Robust.Benchmarks/Exporters/SQLExporter.cs new file mode 100644 index 000000000..3f44b6615 --- /dev/null +++ b/Robust.Benchmarks/Exporters/SQLExporter.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Reports; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Npgsql; + +namespace Robust.Benchmarks.Exporters; + +public sealed class SQLExporter : IExporter +{ + public static readonly IExporter Default = new SQLExporter(); + + private SQLExporter(){} + + public void ExportToLog(Summary summary, ILogger logger) + { + Export(summary, logger); + } + + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + Export(summary, consoleLogger); + return Array.Empty(); + } + + private bool TryGetEnvironmentVariable(string name, ILogger logger, [NotNullWhen(true)] out string? value) + { + value = Environment.GetEnvironmentVariable("name"); + if (value == null) + logger.WriteError($"ROBUST_BENCHMARKS_ENABLE_SQL is set, but {name} is missing."); + return value != null; + } + + private void Export(Summary summary, ILogger logger) + { + if (!TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_ADDRESS", logger, out var address) || + !TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_PORT", logger, out var rawPort) || + !TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_USER", logger, out var user) || + !TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_PASSWORD", logger, out var password) || + !TryGetEnvironmentVariable("ROBUST_BENCHMARKS_SQL_DATABASE", logger, out var db) || + !TryGetEnvironmentVariable("GITHUB_SHA", logger, out var gitHash)) + return; + + if (!int.TryParse(rawPort, out var port)) + { + logger.WriteError("Failed parsing ROBUST_BENCHMARKS_SQL_PORT to int."); + return; + } + + var builder = new DbContextOptionsBuilder(); + var connectionString = new NpgsqlConnectionStringBuilder + { + Host = address, + Port = port, + Database = db, + Username = user, + Password = password + }.ConnectionString; + builder.UseNpgsql(connectionString); + using var ctx = new BenchmarkContext(builder.Options); + try + { + ctx.Database.Migrate(); + ctx.BenchmarkRuns.Add(BenchmarkRun.FromSummary(summary, gitHash)); + ctx.SaveChanges(); + } + finally + { + ctx.Dispose(); + } + } + + public string Name => "sql"; +} + +public sealed class DesignTimeContextFactoryPostgres : IDesignTimeDbContextFactory +{ + public BenchmarkContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseNpgsql("Server=localhost"); + return new BenchmarkContext(optionsBuilder.Options); + } +} + +public class BenchmarkContext : DbContext +{ + public DbSet BenchmarkRuns { get; set; } = default!; + + public BenchmarkContext() { } + public BenchmarkContext(DbContextOptions options) : base(options) { } +} + +public class BenchmarkRun +{ + [Key, DatabaseGenerated(DatabaseGeneratedOption.Identity)] + public ulong Id { get; set; } + public string GitHash { get; set; } = string.Empty; + [Column(TypeName = "timestamptz")] + public DateTime RunDate { get; set; } + public string Name { get; set; } = string.Empty; + [Column(TypeName = "jsonb")] + public BenchmarkRunReport[] Reports { get; set; } = Array.Empty(); + + public static BenchmarkRun FromSummary(Summary summary, string gitHash) + { + return new BenchmarkRun + { + Reports = summary.Reports.Select(r => new BenchmarkRunReport + { + Parameters = r.BenchmarkCase.Parameters.Items.Select(p => new BenchmarkRunParameter + { + Name = p.Name, + Value = p.Value + }).ToArray(), + Statistics = r.ResultStatistics + }).ToArray(), + Name = summary.BenchmarksCases.First().FolderInfo, + RunDate = DateTime.UtcNow, + GitHash = gitHash + }; + } +} + +public class BenchmarkRunReport +{ + public BenchmarkRunParameter[] Parameters { get; set; } = Array.Empty(); + public Statistics Statistics { get; set; } = default!; +} + +public class BenchmarkRunParameter +{ + public string Name { get; set; } = string.Empty; + public object Value { get; set; } = default!; +} diff --git a/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.Designer.cs b/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.Designer.cs new file mode 100644 index 000000000..e4cff1c6f --- /dev/null +++ b/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.Designer.cs @@ -0,0 +1,55 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Robust.Benchmarks.Exporters; + +#nullable disable + +namespace Robust.Benchmarks.Migrations +{ + [DbContext(typeof(BenchmarkContext))] + [Migration("20220328231938_InitialCreate")] + partial class InitialCreate + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Robust.Benchmarks.Exporters.BenchmarkRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(20,0)"); + + b.Property("GitHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Reports") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("RunDate") + .HasColumnType("Date"); + + b.HasKey("Id"); + + b.ToTable("BenchmarkRuns"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.cs b/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.cs new file mode 100644 index 000000000..73d7a6a37 --- /dev/null +++ b/Robust.Benchmarks/Migrations/20220328231938_InitialCreate.cs @@ -0,0 +1,35 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Robust.Benchmarks.Exporters; + +#nullable disable + +namespace Robust.Benchmarks.Migrations +{ + public partial class InitialCreate : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "BenchmarkRuns", + columns: table => new + { + Id = table.Column(type: "numeric(20,0)", nullable: false), + GitHash = table.Column(type: "text", nullable: false), + RunDate = table.Column(type: "Date", nullable: false), + Name = table.Column(type: "text", nullable: false), + Reports = table.Column(type: "jsonb", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_BenchmarkRuns", x => x.Id); + }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "BenchmarkRuns"); + } + } +} diff --git a/Robust.Benchmarks/Migrations/BenchmarkContextModelSnapshot.cs b/Robust.Benchmarks/Migrations/BenchmarkContextModelSnapshot.cs new file mode 100644 index 000000000..da90a6c42 --- /dev/null +++ b/Robust.Benchmarks/Migrations/BenchmarkContextModelSnapshot.cs @@ -0,0 +1,53 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Robust.Benchmarks.Exporters; + +#nullable disable + +namespace Robust.Benchmarks.Migrations +{ + [DbContext(typeof(BenchmarkContext))] + partial class BenchmarkContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("Robust.Benchmarks.Exporters.BenchmarkRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("numeric(20,0)"); + + b.Property("GitHash") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("Reports") + .IsRequired() + .HasColumnType("jsonb"); + + b.Property("RunDate") + .HasColumnType("Date"); + + b.HasKey("Id"); + + b.ToTable("BenchmarkRuns"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Robust.Benchmarks/NumericsHelpers/AddBenchmark.cs b/Robust.Benchmarks/NumericsHelpers/AddBenchmark.cs index 3cec3781d..b6f319543 100644 --- a/Robust.Benchmarks/NumericsHelpers/AddBenchmark.cs +++ b/Robust.Benchmarks/NumericsHelpers/AddBenchmark.cs @@ -9,6 +9,9 @@ namespace Robust.Benchmarks.NumericsHelpers [Params(32, 128)] public int N { get; set; } + [Params(1,2)] + public int T { get; set; } + private float[] _inputA = default!; private float[] _inputB = default!; private float[] _output = default!; diff --git a/Robust.Benchmarks/Program.cs b/Robust.Benchmarks/Program.cs index fced03366..458f50494 100644 --- a/Robust.Benchmarks/Program.cs +++ b/Robust.Benchmarks/Program.cs @@ -1,6 +1,8 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Running; using System; +using Robust.Benchmarks.Configs; +using Robust.Benchmarks.Exporters; namespace Robust.Benchmarks { @@ -16,7 +18,8 @@ namespace Robust.Benchmarks Console.WriteLine("THE DEBUG BUILD IS ONLY GOOD FOR FIXING A CRASHING BENCHMARK\n"); BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig()); #else - BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + var config = Environment.GetEnvironmentVariable("ROBUST_BENCHMARKS_ENABLE_SQL") != null ? DefaultSQLConfig.Instance : null; + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); #endif } } diff --git a/Robust.Benchmarks/Robust.Benchmarks.csproj b/Robust.Benchmarks/Robust.Benchmarks.csproj index 3b7b5968a..fb56c4d51 100644 --- a/Robust.Benchmarks/Robust.Benchmarks.csproj +++ b/Robust.Benchmarks/Robust.Benchmarks.csproj @@ -14,6 +14,11 @@ + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + diff --git a/Robust.Benchmarks/add-migration.ps1 b/Robust.Benchmarks/add-migration.ps1 new file mode 100644 index 000000000..11ea18800 --- /dev/null +++ b/Robust.Benchmarks/add-migration.ps1 @@ -0,0 +1,11 @@ +#!/usr/bin/env pwsh + +param([String]$name) + +if ($name -eq "") +{ + Write-Error "must specify migration name" + exit +} + +dotnet ef migrations add --context BenchmarkContext -o Migrations $name diff --git a/Robust.Benchmarks/add-migration.sh b/Robust.Benchmarks/add-migration.sh new file mode 100644 index 000000000..9d2778dc7 --- /dev/null +++ b/Robust.Benchmarks/add-migration.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +if [ -z "$1" ] ; then + echo "Must specify migration name" + exit 1 +fi + +dotnet ef migrations add --context BenchmarkContext -o Migrations "$1"