"Add vorbis" she said. "It'll be easy"

Yeah so this adds libogg, libvorbis, and libopus to the Rust library on the client. This is intended for use by the client soon-ish to replace .NET ogg vorbis implementations and add opus support.

This turns out to be a huge pain thanks to https://github.com/rust-lang/rfcs/issues/2771 . To solve this I ended up compiling the projects as staticlib and creating a build.py script to invoke the linker. This sucks a *lot* and I have yet to write the linker invocations for Linux/Windows, but it's probably the best option we have.

native/Cargo.lock

@@ -25,9 +25,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.45"
+version = "1.2.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe"
+checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -36,6 +36,7 @@ dependencies = [
 [[package]]
 name = "cef"
 version = "141.6.0+141.0.11"
+source = "git+https://github.com/space-wizards/cef-rs.git?rev=10afdf90bcafb0491adc4ef535530eeee77f4ef9#10afdf90bcafb0491adc4ef535530eeee77f4ef9"
 dependencies = [
  "cef-dll-sys",
  "libloading",
@@ -46,6 +47,7 @@ dependencies = [
 [[package]]
 name = "cef-dll-sys"
 version = "141.6.0+141.0.11"
+source = "git+https://github.com/space-wizards/cef-rs.git?rev=10afdf90bcafb0491adc4ef535530eeee77f4ef9#10afdf90bcafb0491adc4ef535530eeee77f4ef9"
 dependencies = [
  "anyhow",
  "cmake",
@@ -67,9 +69,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
 name = "cmake"
-version = "0.1.54"
+version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
+checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
 dependencies = [
  "cc",
 ]
@@ -87,21 +89,22 @@ dependencies = [
 [[package]]
 name = "download-cef"
 version = "2.2.0"
+source = "git+https://github.com/space-wizards/cef-rs.git?rev=10afdf90bcafb0491adc4ef535530eeee77f4ef9#10afdf90bcafb0491adc4ef535530eeee77f4ef9"
 dependencies = [
  "thiserror",
 ]
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.4"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
+checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff"
 
 [[package]]
 name = "libc"
-version = "0.2.177"
+version = "0.2.179"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
+checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f"
 
 [[package]]
 name = "libloading"
@@ -113,6 +116,25 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "mimalloc"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "objc2"
 version = "0.6.3"
@@ -265,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
+checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
 dependencies = [
  "unicode-ident",
 ]
@@ -288,6 +310,26 @@ dependencies = [
  "cef",
 ]
 
+[[package]]
+name = "robust-native-client"
+version = "0.1.0"
+dependencies = [
+ "mimalloc",
+ "robust-native-shared",
+]
+
+[[package]]
+name = "robust-native-server"
+version = "0.1.0"
+dependencies = [
+ "mimalloc",
+ "robust-native-shared",
+]
+
+[[package]]
+name = "robust-native-shared"
+version = "0.1.0"
+
 [[package]]
 name = "robust-native-webview"
 version = "0.1.0"
@@ -306,9 +348,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "syn"
-version = "2.0.109"
+version = "2.0.113"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f"
+checksum = "678faa00651c9eb72dd2020cbdf275d92eccb2400d568e419efdd64838145cb4"
 dependencies = [
  "proc-macro2",
  "quote",

native/Cargo.toml

@@ -1,9 +1,10 @@
 [workspace]
 resolver = "3"
-members = ["cef-helper", "robust-apphost", "robust-native-webview"]
+members = ["cef-helper", "robust-apphost", "robust-native-client", "robust-native-server", "robust-native-shared", "robust-native-webview"]
 
 [workspace.dependencies]
-cef = { path = "../../../cef-rs/cef", default-features = false }
+cef = { git = "https://github.com/space-wizards/cef-rs.git", rev = "10afdf90bcafb0491adc4ef535530eeee77f4ef9", default-features = false }
 objc2 = "0.6.3"
 objc2-app-kit = "0.3.2"
 cc = "1.0"
+mimalloc = "0.1.48"

native/build.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+#
+# This helper exists SOLELY to work around https://github.com/rust-lang/rfcs/issues/2771
+# Instead of compiling our libs directly to a cdylib, we compile them to a staticlib
+# and link manually. This is cool and all, except that's a lot of work.
+# And involves three platform-specific linkers. *sigh*
+#
+
+import argparse
+import os
+import subprocess
+import platform
+import shlex
+import tempfile
+
+from dataclasses import dataclass, field
+
+@dataclass
+class LinkerData:
+    out_file: str
+    inputs: list[str] = field(default_factory=list)
+    libs: list[str] = field(default_factory=list)
+    lib_search_paths: list[str] = field(default_factory=list)
+    pkg_config_libs: list[str] = field(default_factory=list)
+    export_symbols: list[str] = field(default_factory=list)
+
+os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--release", action="store_true")
+
+    args = parser.parse_args()
+    cmd = ["cargo", "build", "-p", "robust-native-server", "-p", "robust-native-client"]
+    if args.release:
+        cmd.append("--release")
+
+    subprocess.run(cmd, check=True)
+
+    target_dir = os.path.join("target", "debug")
+    if args.release:
+        target_dir = os.path.join("target", "release")
+
+    link(LinkerData(
+        out_file=os.path.join(target_dir, platform_dylib_name("robust_native_client")),
+        inputs=[os.path.join(target_dir, platform_staticlib_name("robust_native_client"))],
+        pkg_config_libs=["ogg", "opus", "vorbis"],
+        export_symbols=read_symbols_def("ogg")
+            + read_symbols_def("vorbis")
+            + read_symbols_def("opus")
+            + read_symbols_def("client")))
+
+    link(LinkerData(
+        out_file=os.path.join(target_dir, platform_dylib_name("robust_native_server")),
+        inputs=[os.path.join(target_dir, platform_staticlib_name("robust_native_server"))]))
+
+def link(data: LinkerData):
+    system = platform.system()
+    if system == "Darwin":
+        link_macos(data)
+    else:
+        raise NotImplementedError()
+
+#
+# Per-platform linker invocations are mostly inspired by what rustc does
+#
+
+def link_macos(data: LinkerData):
+    symbol_list_file = tempfile.NamedTemporaryFile("w+")
+    symbol_list_file.writelines((f"_{n.strip()}\n" for n in data.export_symbols))
+    symbol_list_file.flush()
+
+    # Todo: crosscompile
+    args = [
+        "cc",
+        "-Wl,-exported_symbols_list",
+        f"-Wl,{symbol_list_file.name}",
+        "-arch", "arm64",
+        "-nodefaultlibs",
+        "-Wl,-dead_strip",
+        "-mmacosx-version-min=11.0.0",
+        "-dynamiclib",
+        "-o", data.out_file,
+        "-liconv", "-lSystem", "-lc", "-lm"
+    ] + get_pkg_config_linker_flags(data.pkg_config_libs) + data.inputs
+
+    args.extend(("-L" + p for p in data.lib_search_paths))
+    args.extend(("-l" + l for l in data.libs))
+
+    subprocess.run(args, check=True)
+
+def get_pkg_config_linker_flags(names: list[str]) -> list[str]:
+    if not names:
+        return []
+
+    result = subprocess.run(["pkg-config", "--libs"] + names, check=True, stdout=subprocess.PIPE)
+    return shlex.split(result.stdout.decode("utf-8"))
+
+def read_symbols_def(name: str) -> list[str]:
+    with open(os.path.join("symbols", name + ".txt"), "r") as f:
+        return [line.strip() for line in f.readlines() if not line.startswith("#")]
+
+def platform_staticlib_name(name: str) -> str:
+    if platform.system() == "Windows":
+        return name + ".lib"
+
+    return f"lib{name}.a"
+
+def platform_dylib_name(name: str) -> str:
+    if platform.system() == "Windows":
+        return name + ".dll"
+
+    if platform.system() == "Darwin":
+        return f"lib{name}.dylib"
+
+    return f"lib{name}.so"
+
+main()

native/robust-native-client/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "robust-native-client"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+mimalloc = { workspace = true }
+robust-native-shared = { path = "../robust-native-shared" }

native/robust-native-client/src/lib.rs

@@ -0,0 +1,4 @@
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;

native/robust-native-server/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "robust-native-server"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+mimalloc = { workspace = true }
+robust-native-shared = { path = "../robust-native-shared" }

native/robust-native-server/src/lib.rs

@@ -0,0 +1,4 @@
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+static GLOBAL: MiMalloc = MiMalloc;

native/robust-native-shared/Cargo.toml

@@ -0,0 +1,4 @@
+[package]
+name = "robust-native-shared"
+version = "0.1.0"
+edition = "2024"

native/symbols/ogg.txt

@@ -0,0 +1,71 @@
+oggpack_writeinit
+oggpack_writecheck
+oggpack_writetrunc
+oggpack_writealign
+oggpack_writecopy
+oggpack_reset
+oggpack_writeclear
+oggpack_readinit
+oggpack_write
+oggpack_look
+oggpack_look1
+oggpack_adv
+oggpack_adv1
+oggpack_read
+oggpack_read1
+oggpack_bytes
+oggpack_bits
+oggpack_get_buffer
+oggpackB_writeinit
+oggpackB_writecheck
+oggpackB_writetrunc
+oggpackB_writealign
+oggpackB_writecopy
+oggpackB_reset
+oggpackB_writeclear
+oggpackB_readinit
+oggpackB_write
+oggpackB_look
+oggpackB_look1
+oggpackB_adv
+oggpackB_adv1
+oggpackB_read
+oggpackB_read1
+oggpackB_bytes
+oggpackB_bits
+oggpackB_get_buffer
+ogg_stream_packetin
+ogg_stream_iovecin
+ogg_stream_pageout
+ogg_stream_pageout_fill
+ogg_stream_flush
+ogg_stream_flush_fill
+ogg_sync_init
+ogg_sync_clear
+ogg_sync_reset
+ogg_sync_destroy
+ogg_sync_check
+ogg_sync_buffer
+ogg_sync_wrote
+ogg_sync_pageseek
+ogg_sync_pageout
+ogg_stream_pagein
+ogg_stream_packetout
+ogg_stream_packetpeek
+ogg_stream_init
+ogg_stream_clear
+ogg_stream_reset
+ogg_stream_reset_serialno
+ogg_stream_destroy
+ogg_stream_check
+ogg_stream_eos
+ogg_page_checksum_set
+ogg_page_version
+ogg_page_continued
+ogg_page_bos
+ogg_page_eos
+ogg_page_granulepos
+ogg_page_serialno
+ogg_page_pageno
+ogg_page_packets
+ogg_packet_clear

native/symbols/opus.txt

@@ -0,0 +1,53 @@
+opus_strerror
+opus_get_version_string
+opus_encoder_get_size
+opus_encoder_create
+opus_encoder_init
+opus_encode
+opus_encode_float
+opus_encoder_destroy
+opus_encoder_ctl
+opus_decoder_get_size
+opus_decoder_create
+opus_decoder_init
+opus_decode
+opus_decode_float
+opus_decoder_ctl
+opus_decoder_destroy
+opus_packet_parse
+opus_packet_get_bandwidth
+opus_packet_get_samples_per_frame
+opus_packet_get_nb_channels
+opus_packet_get_nb_frames
+opus_packet_get_nb_samples
+opus_decoder_get_nb_samples
+opus_pcm_soft_clip
+opus_repacketizer_get_size
+opus_repacketizer_init
+opus_repacketizer_create
+opus_repacketizer_destroy
+opus_repacketizer_cat
+opus_repacketizer_out_range
+opus_repacketizer_get_nb_frames
+opus_repacketizer_out
+opus_packet_pad
+opus_packet_unpad
+opus_multistream_packet_pad
+opus_multistream_packet_unpad
+opus_multistream_encoder_get_size
+opus_multistream_surround_encoder_get_size
+opus_multistream_encoder_create
+opus_multistream_surround_encoder_create
+opus_multistream_encoder_init
+opus_multistream_surround_encoder_init
+opus_multistream_encode
+opus_multistream_encode_float
+opus_multistream_encoder_destroy
+opus_multistream_encoder_ctl
+opus_multistream_decoder_get_size
+opus_multistream_decoder_create
+opus_multistream_decoder_init
+opus_multistream_decode
+opus_multistream_decode_float
+opus_multistream_decoder_ctl
+opus_multistream_decoder_destroy

native/symbols/vorbis.txt

@@ -0,0 +1,36 @@
+vorbis_info_init
+vorbis_info_clear
+vorbis_info_blocksize
+vorbis_comment_init
+vorbis_comment_add
+vorbis_comment_add_tag
+vorbis_comment_query
+vorbis_comment_query_count
+vorbis_comment_clear
+vorbis_block_init
+vorbis_block_clear
+vorbis_dsp_clear
+vorbis_granule_time
+vorbis_version_string
+vorbis_analysis_init
+vorbis_commentheader_out
+vorbis_analysis_headerout
+vorbis_analysis_buffer
+vorbis_analysis_wrote
+vorbis_analysis_blockout
+vorbis_analysis
+vorbis_bitrate_addblock
+vorbis_bitrate_flushpacket
+vorbis_synthesis_idheader
+vorbis_synthesis_headerin
+vorbis_synthesis_init
+vorbis_synthesis_restart
+vorbis_synthesis
+vorbis_synthesis_trackonly
+vorbis_synthesis_blockin
+vorbis_synthesis_pcmout
+vorbis_synthesis_lapout
+vorbis_synthesis_read
+vorbis_packet_blocksize
+vorbis_synthesis_halfrate
+vorbis_synthesis_halfrate_p