From dc67a671678e42f2b73c55677373fc5238693438 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Thu, 15 Oct 2026 16:51:51 -0700 Subject: [PATCH] Link the native library as a shared library in development builds make dev composed the 1.6 GB debug static archive on every run, copied it twice, hashed it twice and statically linked it into the Go binary: a no-op make dev took about 19 seconds. Development builds (Meson's shared default_library) now compose a cdylib instead, which the Go binary loads at run time, as it did libiroh_streamplace.so before. Shipped and cross builds stay static. - The composition only runs when Cargo rebuilt an rlib or the composition changed, and its outputs are hard-linked, not copied. - The Iroh Go bindings only regenerate when Cargo relinks the library the generator reads. - A shared library needs no archive digest to make Go relink, so the generated digest file is gone and hack/native-go.sh passes through. - make native, which nothing called and which would now overwrite the development library, is gone. A no-op make dev is now about 2 seconds, and a Rust-only change relinks the library without relinking the Go binary. --- Makefile | 26 ++++----- hack/build-native.sh | 90 ++++++++++++++++++++--------- hack/native-go.sh | 6 +- rust/streamplace-native/meson.build | 15 ++++- 4 files changed, 95 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index 44aebf459..0602bd09b 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ MACOS_VERSION_FLAG= ifeq ($(BUILDOS),darwin) export MACOSX_DEPLOYMENT_TARGET MACOS_VERSION_FLAG=-mmacosx-version-min=$(MACOSX_DEPLOYMENT_TARGET) + NATIVE_DYLIB_EXT=dylib +else + NATIVE_DYLIB_EXT=so endif BUILDDIR?=build-$(BUILDOS)-$(BUILDARCH) PKG_CONFIG_PATH=$(shell pwd)/$(BUILDDIR)/meson-uninstalled:$(shell pwd)/$(BUILDDIR)/lib/pkgconfig:$(shell pwd)/$(BUILDDIR)/lib/gstreamer-1.0/pkgconfig @@ -35,8 +38,6 @@ ifeq ($(BUILDOS),windows) else ifneq ($(BUILDOS),darwin) NATIVE_CGO_LDFLAGS+= -Wl,-Bstatic -lstdc++ -Wl,-Bdynamic endif -NATIVE_PROFILE?=debug -RUST_TRIPLE?= # Sentinel file that records when lexicons were last built LEXICON_STAMP := .build/lexicon-stamp @@ -290,10 +291,6 @@ meson-compile: native-source: meson subprojects download --sourcedir . moq -.PHONY: native -native: native-source - bash hack/build-native.sh "$(CURDIR)" "$(NATIVE_LIBDIR)" "$(NATIVE_PROFILE)" "$(RUST_TRIPLE)" - # _____ _______ _______ _____ _____ # / ____|__ __|/\|__ __|_ _/ ____| @@ -390,14 +387,15 @@ dev-setup-meson-configure: .PHONY: dev-rust dev-rust: native-source .build/bin/uniffi-bindgen-go-forked meson compile -C $(BUILDDIR) streamplace-native - EXT=so; \ - if [ "$(BUILDOS)" = "darwin" ]; then EXT=dylib; fi; \ - .build/bin/uniffi-bindgen-go-forked --crate iroh_streamplace --out-dir pkg/iroh/generated --library "$(NATIVE_LIBDIR)/libiroh_streamplace.$$EXT" \ - && mkdir -p $(BUILDDIR)/lib/ \ - && cp "$(NATIVE_LIBDIR)/libstreamplace_native.a" $(BUILDDIR)/lib/libstreamplace_native.a.tmp \ - && mv $(BUILDDIR)/lib/libstreamplace_native.a.tmp $(BUILDDIR)/lib/libstreamplace_native.a \ - && cp "$(NATIVE_LIBDIR)/libmoq_ffi.a" $(BUILDDIR)/lib/libmoq_ffi.a.tmp \ - && mv $(BUILDDIR)/lib/libmoq_ffi.a.tmp $(BUILDDIR)/lib/libmoq_ffi.a + $(MAKE) $(IROH_GO_BINDINGS) + mkdir -p $(BUILDDIR)/lib/ + ln -f "$(NATIVE_LIBDIR)/libstreamplace_native.$(NATIVE_DYLIB_EXT)" $(BUILDDIR)/lib/ + +# The generator reads the whole library, so it only runs when Cargo relinks it. +IROH_GO_BINDINGS=pkg/iroh/generated/iroh_streamplace/iroh_streamplace.go +$(IROH_GO_BINDINGS): $(NATIVE_LIBDIR)/libiroh_streamplace.$(NATIVE_DYLIB_EXT) + .build/bin/uniffi-bindgen-go-forked --crate iroh_streamplace --out-dir pkg/iroh/generated --library "$<" + touch $@ .PHONY: dev-test dev-test: diff --git a/hack/build-native.sh b/hack/build-native.sh index 9fbf1c587..7250e5685 100755 --- a/hack/build-native.sh +++ b/hack/build-native.sh @@ -9,6 +9,7 @@ mkdir -p "$2" output_dir=$(cd "$2" && pwd -P) profile=${3:-release} triple=${4:-} +crate_type=${5:-staticlib} # Resolve the rustup proxy once, before either Cargo invocation. Both graphs and # the final staticlib must use the same compiler, including its standard library. @@ -49,32 +50,69 @@ cargo "${cargo_args[@]}" --manifest-path "$source_dir/Cargo.toml" \ cargo "${cargo_args[@]}" --manifest-path "$source_dir/subprojects/moq/Cargo.toml" \ --package moq-ffi --target-dir "$output_dir/moq" -# Native C/C++ static dependencies are bundled in the rlibs; target system libs -# are propagated by Meson's streamplacedeps.pc. Print rustc's complete list too, -# so a dependency change can be checked against that explicit system-lib list. -# With --target, proc-macro dylibs live in the separate host profile/deps -# directories. rustc needs those too when loading either rlib's crate metadata. -"$RUSTC" --edition=2021 --crate-name streamplace_native --crate-type staticlib \ - --target "$triple" "${rustflags[@]}" --print native-static-libs \ - -L "dependency=$output_dir/iroh/$triple/$profile/deps" \ - -L "dependency=$output_dir/moq/$triple/$profile/deps" \ - -L "dependency=$output_dir/iroh/$profile/deps" \ - -L "dependency=$output_dir/moq/$profile/deps" \ - --extern "iroh_streamplace=$output_dir/iroh/$triple/$profile/libiroh_streamplace.rlib" \ - --extern "moq_ffi=$output_dir/moq/$triple/$profile/libmoq_ffi.rlib" \ - "$source_dir/rust/streamplace-native/src/lib.rs" \ - -o "$output_dir/libstreamplace_native.a.tmp" -mv "$output_dir/libstreamplace_native.a.tmp" "$output_dir/libstreamplace_native.a" -# Both names resolve to the same archive/object identities, not two runtimes. -ln -f "$output_dir/libstreamplace_native.a" "$output_dir/libmoq_ffi.a.tmp" -mv "$output_dir/libmoq_ffi.a.tmp" "$output_dir/libmoq_ffi.a" -# The old generator only reads Iroh's UniFFI 0.28 metadata, never MoQ's 0.32. case "$triple" in - *-apple-darwin) ext=dylib ;; - *-linux-*) ext=so ;; - *) ext= ;; + *-apple-darwin) dylib_ext=dylib ;; + *-linux-*) dylib_ext=so ;; + *) dylib_ext= ;; +esac +compose_args=() +case "$crate_type" in + staticlib) ext=a ;; + cdylib) + ext=$dylib_ext + # One library under both names: link-time lookups of either resolve to + # this file, recorded once by its own name. + if [[ "$ext" == so ]]; then + compose_args+=(-C link-arg=-Wl,-soname,libstreamplace_native.so) + else + compose_args+=(-C "link-arg=-Wl,-install_name,$output_dir/libstreamplace_native.dylib") + fi + ;; + *) + echo "unsupported native crate type: $crate_type" >&2 + exit 1 + ;; esac -if [[ -n "$ext" ]]; then - cp "$output_dir/iroh/$triple/$profile/libiroh_streamplace.$ext" \ - "$output_dir/libiroh_streamplace.$ext" +out="$output_dir/libstreamplace_native.$ext" +iroh_rlib="$output_dir/iroh/$triple/$profile/libiroh_streamplace.rlib" +moq_rlib="$output_dir/moq/$triple/$profile/libmoq_ffi.rlib" + +# Composing is a full link of both graphs, so it only runs when Cargo rebuilt +# an rlib or the composition itself changed. +fresh=true +for input in "$iroh_rlib" "$moq_rlib" "$source_dir/rust/streamplace-native/src/lib.rs" "${BASH_SOURCE[0]}"; do + [[ "$out" -nt "$input" ]] || fresh=false +done +if [[ "$fresh" != true ]]; then + # Native C/C++ static dependencies are bundled in the rlibs; target system + # libs are propagated by Meson's streamplacedeps.pc. A staticlib also prints + # rustc's complete list, so a dependency change can be checked against that + # explicit system-lib list. With --target, proc-macro dylibs live in the + # separate host profile/deps directories. rustc needs those too when loading + # either rlib's crate metadata. + "$RUSTC" --edition=2021 --crate-name streamplace_native --crate-type "$crate_type" \ + --target "$triple" "${rustflags[@]}" ${compose_args[@]+"${compose_args[@]}"} --print native-static-libs \ + -L "dependency=$output_dir/iroh/$triple/$profile/deps" \ + -L "dependency=$output_dir/moq/$triple/$profile/deps" \ + -L "dependency=$output_dir/iroh/$profile/deps" \ + -L "dependency=$output_dir/moq/$profile/deps" \ + --extern "iroh_streamplace=$iroh_rlib" \ + --extern "moq_ffi=$moq_rlib" \ + "$source_dir/rust/streamplace-native/src/lib.rs" \ + -o "$out.tmp" + mv "$out.tmp" "$out" + # Both names resolve to the same objects, not two runtimes. + ln -f "$out" "$output_dir/libmoq_ffi.$ext.tmp" + mv "$output_dir/libmoq_ffi.$ext.tmp" "$output_dir/libmoq_ffi.$ext" +fi +# The other kind, from a build of the other type, must not shadow this one. +for stale in a "$dylib_ext"; do + if [[ -n "$stale" && "$stale" != "$ext" ]]; then + rm -f "$output_dir/libstreamplace_native.$stale" "$output_dir/libmoq_ffi.$stale" + fi +done +# The old generator only reads Iroh's UniFFI 0.28 metadata, never MoQ's 0.32. +if [[ -n "$dylib_ext" ]]; then + ln -f "$output_dir/iroh/$triple/$profile/libiroh_streamplace.$dylib_ext" \ + "$output_dir/libiroh_streamplace.$dylib_ext" fi diff --git a/hack/native-go.sh b/hack/native-go.sh index 1f90da3c5..be9aa181d 100755 --- a/hack/native-go.sh +++ b/hack/native-go.sh @@ -3,9 +3,13 @@ set -euo pipefail # Go's cache does not track external static-library contents. Changing a cgo # compiler flag invalidates native-dependent packages and their cached tests, -# without rebuilding unrelated pure-Go packages with a global -a. +# without rebuilding unrelated pure-Go packages with a global -a. A shared +# library (development builds) is loaded at run time, so it needs nothing. archive="$1/libstreamplace_native.a" shift +if [[ ! -f "$archive" ]]; then + exec "$@" +fi if command -v sha256sum >/dev/null 2>&1; then digest=$(sha256sum "$archive") else diff --git a/rust/streamplace-native/meson.build b/rust/streamplace-native/meson.build index ec2a33f1b..478df2a58 100644 --- a/rust/streamplace-native/meson.build +++ b/rust/streamplace-native/meson.build @@ -4,8 +4,20 @@ subproject('moq') rust_triple = meson.get_external_property('RUST_TRIPLE', '') native_profile = 'release' +# Development builds link a shared library: the Go binary loads it at run +# time, so a Go-only change relinks against a small stub, not the whole debug +# Rust graph. Shipped and cross builds stay static. +native_crate_type = 'staticlib' +native_libs_ext = 'a' if get_option('default_library') == 'shared' native_profile = 'debug' + if host_machine.system() == 'linux' + native_crate_type = 'cdylib' + native_libs_ext = 'so' + elif host_machine.system() == 'darwin' + native_crate_type = 'cdylib' + native_libs_ext = 'dylib' + endif endif # Cargo linker/cc-crate settings accept one executable, not an argv array. @@ -49,7 +61,7 @@ endif native_libdir = meson.current_build_dir() native_libs = custom_target( 'streamplace-native', - output: ['libstreamplace_native.a', 'libmoq_ffi.a'], + output: ['libstreamplace_native.' + native_libs_ext, 'libmoq_ffi.' + native_libs_ext], command: [ find_program('bash'), files('../../hack/build-native.sh'), @@ -57,6 +69,7 @@ native_libs = custom_target( native_libdir, native_profile, rust_triple, + native_crate_type, ], env: native_env, build_always_stale: true, # Cargo, not Ninja, tracks the two Rust source graphs. -- knotmirror 0.0.1