Audio rework unrevert + audio packaging (#4555)

Co-authored-by: Pieter-Jan Briers <pieterjan.briers@gmail.com>

Robust.Packaging/AssetProcessing/Passes/AssetPassAudioMetadata.cs

@@ -0,0 +1,83 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
+using Robust.Shared.Audio;
+using Robust.Shared.Audio.AudioLoading;
+using Robust.Shared.Serialization;
+using YamlDotNet.Core;
+using YamlDotNet.RepresentationModel;
+
+namespace Robust.Packaging.AssetProcessing.Passes;
+
+/// <summary>
+/// Strips out audio files and writes them to a metadata .yml
+/// Used for server packaging to avoid bundling entire audio files on the server.
+/// </summary>
+public sealed class AssetPassAudioMetadata : AssetPass
+{
+    private readonly List<AudioMetadataPrototype> _audioMetadata = new();
+    private readonly string _metadataPath;
+
+    public AssetPassAudioMetadata(string metadataPath = "Prototypes/_audio_metadata.yml")
+    {
+        _metadataPath = metadataPath;
+    }
+
+    protected override AssetFileAcceptResult AcceptFile(AssetFile file)
+    {
+        if (!AudioLoader.IsLoadableAudioFile(file.Path))
+            return AssetFileAcceptResult.Pass;
+
+        using var stream = file.Open();
+        var metadata = AudioLoader.LoadAudioMetadata(stream, file.Path);
+
+        lock (_audioMetadata)
+        {
+            _audioMetadata.Add(new AudioMetadataPrototype()
+            {
+                ID = "/" + file.Path,
+                Length = metadata.Length,
+            });
+        }
+
+        return AssetFileAcceptResult.Consumed;
+    }
+
+    [SuppressMessage("ReSharper", "InconsistentlySynchronizedField")]
+    protected override void AcceptFinished()
+    {
+        if (_audioMetadata.Count == 0)
+        {
+            Logger?.Debug("Have no audio metadata, not writing anything");
+            return;
+        }
+
+        Logger?.Debug("Writing audio metadata for {0} audio files", _audioMetadata.Count);
+
+        // ReSharper disable once InconsistentlySynchronizedField
+        var root = new YamlSequenceNode();
+        var document = new YamlDocument(root);
+
+        foreach (var prototype in _audioMetadata)
+        {
+            // TODO: I know but sermanager and please get me out of this hell.
+            var jaml = new YamlMappingNode
+            {
+                { "type", AudioMetadataPrototype.ProtoName },
+                { "id", new YamlScalarNode(prototype.ID) },
+                { "length", new YamlScalarNode(prototype.Length.TotalSeconds.ToString(CultureInfo.InvariantCulture)) }
+            };
+            root.Add(jaml);
+        }
+
+        RunJob(() =>
+        {
+            using var memory = new MemoryStream();
+            using var writer = new StreamWriter(memory);
+            var yamlStream = new YamlStream(document);
+            yamlStream.Save(new YamlNoDocEndDotsFix(new YamlMappingFix(new Emitter(writer))), false);
+            writer.Flush();
+            var result = new AssetFileMemory(_metadataPath, memory.ToArray());
+            SendFile(result);
+        });
+    }
+}

Robust.Packaging/AssetProcessing/Passes/AssetPassPrefix.cs

@@ -0,0 +1,28 @@
+namespace Robust.Packaging.AssetProcessing.Passes;
+
+/// <summary>
+/// Appends a prefix to file paths of passed-through files.
+/// </summary>
+public sealed class AssetPassPrefix : AssetPass
+{
+    public string Prefix { get; set; }
+
+    public AssetPassPrefix(string prefix)
+    {
+        Prefix = prefix;
+    }
+
+    protected override AssetFileAcceptResult AcceptFile(AssetFile file)
+    {
+        var newPath = Prefix + file.Path;
+        var newFile = file switch
+        {
+            AssetFileDisk disk => (AssetFile) new AssetFileDisk(newPath, disk.DiskPath),
+            AssetFileMemory memory => new AssetFileMemory(newPath, memory.Memory),
+            _ => throw new ArgumentOutOfRangeException(nameof(file))
+        };
+
+        SendFile(newFile);
+        return AssetFileAcceptResult.Consumed;
+    }
+}

Robust.Packaging/Robust.Packaging.csproj

@@ -10,5 +10,9 @@
     <ProjectReference Include="..\Robust.Shared\Robust.Shared.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="NVorbis" Version="0.10.5" />
+  </ItemGroup>
+
   <Import Project="..\MSBuild\Robust.Properties.targets" />
 </Project>

Robust.Packaging/RobustClientAssetGraph.cs

@@ -21,6 +21,7 @@ public sealed class RobustClientAssetGraph
     /// </summary>
     public IReadOnlyCollection<AssetPass> AllPasses { get; }
 
+    /// <param name="parallel">Should inputs be run in parallel. Should only be turned off for debugging.</param>
     public RobustClientAssetGraph(bool parallel = true)
     {
         // The code injecting the list of source files is assumed to be pretty single-threaded.
@@ -40,7 +41,7 @@ public sealed class RobustClientAssetGraph
             Input,
             PresetPasses,
             Output,
-            NormalizeText
+            NormalizeText,
         };
     }
 }

Robust.Packaging/RobustServerAssetGraph.cs

@@ -0,0 +1,129 @@
+using Robust.Packaging.AssetProcessing;
+using Robust.Packaging.AssetProcessing.Passes;
+
+namespace Robust.Packaging;
+
+/// <summary>
+/// Standard asset graph for packaging server files. Extend by wiring things up to <see cref="InputCore"/>, <see cref="InputResources"/>, and <see cref="Output"/>.
+/// </summary>
+/// <remarks>
+/// <para>
+/// This graph has two inputs: one for "core" server files such as the main engine executable, and another for resource files.
+/// </para>
+/// <para>
+/// If you want to add extra passes to run before preset passes, depend them on the relevant input pass, with a before of the relevant preset pass.
+/// </para>
+/// <para>
+/// See the following graph (Mermaid syntax) to understand this asset graph:
+/// </para>
+/// <code>
+/// flowchart LR
+///     InputCore              -->    PresetPassesCore
+///     PresetPassesCore       --1--> NormalizeTextCore
+///     NormalizeTextCore      -->    Output
+///     PresetPassesCore       --2--> Output
+///     InputResources         -->    PresetPassesResources
+///     PresetPassesResources  --1--> AudioMetadata
+///     PresetPassesResources  --2--> NormalizeTextResources
+///     PresetPassesResources  --3--> PrefixResources
+///     AudioMetadata          -->    PrefixResources
+///     NormalizeTextResources -->    PrefixResources
+///     PrefixResources        -->    Output
+/// </code>
+/// </remarks>
+public sealed class RobustServerAssetGraph
+{
+    public AssetPassPipe Output { get; }
+
+    /// <summary>
+    /// Input pass for core server files, such as <c>Robust.Server.exe</c>.
+    /// </summary>
+    /// <seealso cref="InputResources"/>
+    public AssetPassPipe InputCore { get; }
+
+    public AssetPassPipe PresetPassesCore { get; }
+
+    /// <summary>
+    /// Normalizes text files in core files.
+    /// </summary>
+    public AssetPassNormalizeText NormalizeTextCore { get; }
+
+    /// <summary>
+    /// Input pass for server resource files. Everything that will go into <c>Resources/</c>.
+    /// </summary>
+    /// <remarks>
+    /// Do not prefix file paths with <c>Resources/</c>, the asset pass will automatically remap them.
+    /// </remarks>
+    /// <seealso cref="InputCore"/>
+    public AssetPassPipe InputResources { get; }
+    public AssetPassPipe PresetPassesResources { get; }
+    public AssetPassAudioMetadata AudioMetadata { get; }
+
+    /// <summary>
+    /// Normalizes text files in resources.
+    /// </summary>
+    public AssetPassNormalizeText NormalizeTextResources { get; }
+
+    /// <summary>
+    /// Responsible for putting resources into the "<c>Resources/</c>" folder.
+    /// </summary>
+    public AssetPassPrefix PrefixResources { get; }
+
+    /// <summary>
+    /// Collection of all passes in this preset graph.
+    /// </summary>
+    public IReadOnlyCollection<AssetPass> AllPasses { get; }
+
+    /// <param name="parallel">Should inputs be run in parallel. Should only be turned off for debugging.</param>
+    public RobustServerAssetGraph(bool parallel = true)
+    {
+        Output = new AssetPassPipe { Name = "RobustServerAssetGraphOutput", CheckDuplicates = true };
+
+        //
+        // Core files
+        //
+
+        // The code injecting the list of source files is assumed to be pretty single-threaded.
+        // We use a parallelizing input to break out all the work on files coming in onto multiple threads.
+        InputCore = new AssetPassPipe { Name = "RobustServerAssetGraphInputCore", Parallelize = parallel };
+        PresetPassesCore = new AssetPassPipe { Name = "RobustServerAssetGraphPresetPassesCore" };
+        NormalizeTextCore = new AssetPassNormalizeText { Name = "RobustServerAssetGraphNormalizeTextCore" };
+
+        PresetPassesCore.AddDependency(InputCore);
+        NormalizeTextCore.AddDependency(PresetPassesCore).AddBefore(Output);
+        Output.AddDependency(PresetPassesCore);
+        Output.AddDependency(NormalizeTextCore);
+
+        //
+        // Resource files
+        //
+
+        // Ditto about parallelizing
+        InputResources = new AssetPassPipe { Name = "RobustServerAssetGraphInputResources", Parallelize = parallel };
+        PresetPassesResources = new AssetPassPipe { Name = "RobustServerAssetGraphPresetPassesResources" };
+        NormalizeTextResources = new AssetPassNormalizeText { Name = "RobustServerAssetGraphNormalizeTextResources" };
+        AudioMetadata = new AssetPassAudioMetadata { Name = "RobustServerAssetGraphAudioMetadata" };
+        PrefixResources = new AssetPassPrefix("Resources/") { Name = "RobustServerAssetGraphPrefixResources" };
+
+        PresetPassesResources.AddDependency(InputResources);
+        AudioMetadata.AddDependency(PresetPassesResources).AddBefore(NormalizeTextResources);
+        NormalizeTextResources.AddDependency(PresetPassesResources).AddBefore(PrefixResources);
+        PrefixResources.AddDependency(PresetPassesResources);
+        PrefixResources.AddDependency(AudioMetadata);
+        PrefixResources.AddDependency(NormalizeTextResources);
+        Output.AddDependency(PrefixResources);
+
+        AllPasses = new AssetPass[]
+        {
+            Output,
+            InputCore,
+            PresetPassesCore,
+            NormalizeTextCore,
+            InputResources,
+            PresetPassesResources,
+            NormalizeTextResources,
+            AudioMetadata,
+            PrefixResources,
+        };
+    }
+}

Robust.Packaging/RobustServerPackaging.cs

@@ -6,7 +6,6 @@ public sealed class RobustServerPackaging
 {
     public static IReadOnlySet<string> ServerIgnoresResources { get; } = new HashSet<string>
     {
-        "Audio",
         "Textures",
         "Fonts",
         "Shaders",
@@ -19,15 +18,16 @@ public sealed class RobustServerPackaging
     {
         var ignoreSet = ServerIgnoresResources.Union(RobustSharedPackaging.SharedIgnoredResources).ToHashSet();
 
-        await RobustSharedPackaging.DoResourceCopy(Path.Combine(contentDir, "Resources"),
+        await RobustSharedPackaging.DoResourceCopy(
+            Path.Combine(contentDir, "Resources"),
             pass,
             ignoreSet,
-            "Resources",
-            cancel);
-        await RobustSharedPackaging.DoResourceCopy(Path.Combine("RobustToolbox", "Resources"),
+            cancel: cancel);
+
+        await RobustSharedPackaging.DoResourceCopy(
+            Path.Combine("RobustToolbox", "Resources"),
             pass,
             ignoreSet,
-            "Resources",
-            cancel);
+            cancel: cancel);
     }
 }