Add asset pass to merge text files in directories.

This massively reduces the file count of published SS14 builds by a few thousand, by combining YAML prototypes and Fluent files in the same folder into one file.

Robust.Packaging/AssetProcessing/AssetPass.cs

@@ -1,4 +1,5 @@
-using Robust.Shared.Analyzers;
+using System.Text;
+using Robust.Shared.Analyzers;
 using Robust.Shared.Collections;
 
 namespace Robust.Packaging.AssetProcessing;
@@ -199,6 +200,17 @@ public class AssetPass
     /// </summary>
     public void InjectFileFromMemory(string path, byte[] memory) => InjectFile(new AssetFileMemory(path, memory));
 
+    /// <summary>
+    /// Convenience method to <see cref="InjectFile"/> a <see cref="AssetFileMemory"/>.
+    /// </summary>
+    public void InjectFileFromMemory(string path, ReadOnlySpan<byte> memory) => InjectFile(new AssetFileMemory(path, memory.ToArray()));
+
+    /// <summary>
+    /// Convenience method to <see cref="InjectFile"/> a <see cref="AssetFileMemory"/> made from text.
+    /// </summary>
+    public void InjectFileFromText(string path, string text) =>
+        InjectFile(new AssetFileMemory(path, Encoding.UTF8.GetBytes(text)));
+
     /// <summary>
     /// Called when all depended-on passes have finished processing, meaning no more files will come in.
     /// </summary>

Robust.Packaging/AssetProcessing/Passes/AssetPassMergeTextDirectories.cs

@@ -0,0 +1,99 @@
+using Robust.Shared.Utility;
+
+namespace Robust.Packaging.AssetProcessing.Passes;
+
+public sealed class AssetPassMergeTextDirectories : AssetPass
+{
+    private readonly ResPath _prefixPath;
+    private readonly string _extension;
+    private readonly Func<string, string>? _formatterHead;
+    private readonly Func<string, string>? _formatterTail;
+
+    private readonly Dictionary<ResPath, DirectoryDatum> _data = new();
+
+    public AssetPassMergeTextDirectories(
+        string prefixPath,
+        string extension,
+        Func<string, string>? formatterHead = null,
+        Func<string, string>? formatterTail = null)
+    {
+        _prefixPath = new ResPath(prefixPath);
+        _extension = extension;
+        _formatterHead = formatterHead;
+        _formatterTail = formatterTail;
+    }
+
+    protected override AssetFileAcceptResult AcceptFile(AssetFile file)
+    {
+        var resPath = new ResPath(file.Path);
+        if (!resPath.TryRelativeTo(_prefixPath, out _))
+            return AssetFileAcceptResult.Pass;
+
+        if (resPath.Extension != _extension)
+            return AssetFileAcceptResult.Pass;
+
+        var directory = resPath.Directory;
+        lock (_data)
+        {
+            var datum = _data.GetOrNew(directory);
+            datum.Files.Add(file);
+        }
+
+        return AssetFileAcceptResult.Consumed;
+    }
+
+    protected override void AcceptFinished()
+    {
+        RunJob(() =>
+        {
+            lock (_data)
+            {
+                var ms = new MemoryStream();
+                var writer = new StreamWriter(ms, EncodingHelpers.UTF8);
+
+                foreach (var (directory, datum) in _data)
+                {
+                    ms.Position = 0;
+                    var mergedFile = directory / $"__merged.{_extension}";
+                    WriteForDatum(datum, writer);
+                    writer.Flush();
+
+                    SendFileFromMemory(mergedFile.ToString(), ms.GetBuffer()[..(int)ms.Position]);
+                }
+
+                _data.Clear();
+            }
+        });
+    }
+
+    private void WriteForDatum(DirectoryDatum datum, StreamWriter writer)
+    {
+        foreach (var file in datum.Files.OrderBy(f => f.Path, StringComparer.Ordinal))
+        {
+            if (_formatterHead != null)
+            {
+                writer.Write(_formatterHead(file.Path));
+                writer.Write('\n');
+            }
+
+            using var stream = file.Open();
+            using var reader = new StreamReader(stream);
+            while (reader.ReadLine() is { } line)
+            {
+                writer.Write(line);
+                writer.Write('\n');
+            }
+
+            if (_formatterTail != null)
+            {
+                writer.Write(_formatterTail(file.Path));
+                writer.Write('\n');
+            }
+        }
+    }
+
+    private sealed class DirectoryDatum
+    {
+        public readonly List<AssetFile> Files = [];
+    }
+}

Robust.UnitTesting/Packaging/AssetPassMergeTextDirectoriesTest.cs

@@ -0,0 +1,76 @@
+using System.Threading.Tasks;
+using NUnit.Framework;
+using Robust.Packaging.AssetProcessing.Passes;
+
+namespace Robust.UnitTesting.Packaging;
+
+[Parallelizable(ParallelScope.All)]
+[TestFixture]
+[TestOf(typeof(AssetPassMergeTextDirectories))]
+internal sealed class AssetPassMergeTextDirectoriesTest
+{
+    [Test]
+    public async Task Test()
+    {
+        var pass = new AssetPassMergeTextDirectories("/Prototypes", "yml", f => $"# BEGIN: {f}", f => $"# END: {f}");
+        var collectorPass = AssetPassTest.SetupTestPass(pass);
+
+        pass.InjectFileFromMemory("/Prototypes/LICENSE.txt", "Do whatever\n"u8);
+        pass.InjectFileFromMemory("/Prototypes/b.yml", "# file: B\n"u8);
+        pass.InjectFileFromMemory("/Prototypes/a.yml", "# file: A\n"u8);
+        pass.InjectFileFromMemory("/Prototypes/z/d.yml", "# file: D\n"u8);
+        pass.InjectFileFromMemory("/Prototypes/z/c.yml", "# file: C\n"u8);
+        pass.InjectFinished();
+        await collectorPass.FinishedTask;
+
+        collectorPass.AssertTextFiles(
+            ("/Prototypes/__merged.yml", """
+                # BEGIN: /Prototypes/a.yml
+                # file: A
+                # END: /Prototypes/a.yml
+                # BEGIN: /Prototypes/b.yml
+                # file: B
+                # END: /Prototypes/b.yml
+
+                """.Replace("\r\n", "\n")),
+            ("/Prototypes/z/__merged.yml", """
+                # BEGIN: /Prototypes/z/c.yml
+                # file: C
+                # END: /Prototypes/z/c.yml
+                # BEGIN: /Prototypes/z/d.yml
+                # file: D
+                # END: /Prototypes/z/d.yml
+
+                """.Replace("\r\n", "\n")));
+    }
+
+    [Test]
+    public async Task TestNormalizeEol()
+    {
+        var pass = new AssetPassMergeTextDirectories("/", "yml");
+        var collectorPass = AssetPassTest.SetupTestPass(pass);
+
+        pass.InjectFileFromMemory("/b.yml", "# file: B\r\n"u8);
+        pass.InjectFileFromMemory("/a.yml", "# file: A\n"u8);
+        pass.InjectFinished();
+        await collectorPass.FinishedTask;
+
+        collectorPass.AssertTextFiles(
+            ("/__merged.yml", "# file: A\n# file: B\n"));
+    }
+
+    [Test]
+    public async Task TestFixBom()
+    {
+        var pass = new AssetPassMergeTextDirectories("/", "yml");
+        var collectorPass = AssetPassTest.SetupTestPass(pass);
+
+        pass.InjectFileFromMemory("/b.yml", "\uFEFF# file: B\n"u8);
+        pass.InjectFileFromMemory("/a.yml", "\uFEFF# file: A\n"u8);
+        pass.InjectFinished();
+        await collectorPass.FinishedTask;
+
+        collectorPass.AssertTextFiles(
+            ("/__merged.yml", "# file: A\n# file: B\n"));
+    }
+}

Robust.UnitTesting/Packaging/AssetPassTest.cs

@@ -0,0 +1,28 @@
+using NUnit.Framework;
+using Robust.Packaging.AssetProcessing;
+
+namespace Robust.UnitTesting.Packaging;
+
+/// <summary>
+/// Helper class for testing <see cref="AssetPass"/>.
+/// </summary>
+public static class AssetPassTest
+{
+    /// <summary>
+    /// Make an asset pass write into a <see cref="AssetPassTestCollector"/> and resolve the graph.
+    /// </summary>
+    /// <remarks>
+    /// The resolved graph logs to the NUnit test context.
+    /// </remarks>
+    public static AssetPassTestCollector SetupTestPass(AssetPass testedPass)
+    {
+        var logger = new PackageLoggerNUnit(TestContext.Out);
+        var collectorPass = new AssetPassTestCollector();
+
+        collectorPass.AddDependency(testedPass);
+
+        AssetGraph.CalculateGraph([testedPass, collectorPass], logger);
+
+        return collectorPass;
+    }
+}

Robust.UnitTesting/Packaging/AssetPassTestCollector.cs

@@ -0,0 +1,55 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NUnit.Framework;
+using Robust.Packaging.AssetProcessing;
+
+namespace Robust.UnitTesting.Packaging;
+
+/// <summary>
+/// A simple asset pass that stores all files it receives, for introspection by tests.
+/// </summary>
+public sealed class AssetPassTestCollector : AssetPass
+{
+    public readonly List<AssetFile> Files = [];
+
+    protected override AssetFileAcceptResult AcceptFile(AssetFile file)
+    {
+        lock (Files)
+        {
+            Files.Add(file);
+        }
+
+        return AssetFileAcceptResult.Consumed;
+    }
+
+    /// <summary>
+    /// Assert that the only files collected are an exact set of test files.
+    /// </summary>
+    public void AssertTextFiles(params (string path, string data)[] files)
+    {
+        lock (Files)
+        {
+            Assert.That(Files, Has.Count.EqualTo(files.Length));
+
+            Assert.Multiple(() =>
+            {
+                foreach (var file in files)
+                {
+                    var matchingFile = Files.SingleOrDefault(f => f.Path == file.path);
+                    if (matchingFile == null)
+                    {
+                        Assert.Fail($"Unable to find file {file.path}");
+                        continue;
+                    }
+
+                    using var fileData = matchingFile.Open();
+                    using var reader = new StreamReader(fileData);
+                    var fileText = reader.ReadToEnd();
+
+                    Assert.That(fileText, Is.EqualTo(file.data));
+                }
+            });
+        }
+    }
+}

Robust.UnitTesting/Packaging/PackageLoggerNUnit.cs

@@ -0,0 +1,17 @@
+using System.IO;
+using Robust.Packaging;
+using Robust.Shared.Log;
+
+namespace Robust.UnitTesting.Packaging;
+
+/// <summary>
+/// Package logger for writing to NUnit's test context.
+/// </summary>
+/// <param name="writer"></param>
+public sealed class PackageLoggerNUnit(TextWriter writer) : IPackageLogger
+{
+    public void Log(LogLevel level, string msg)
+    {
+        writer.WriteLine($"[{level}] {msg}");
+    }
+}