From f647b512efb14b00ac4f2c5c6f92dab1d77ebc15 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 12 Oct 2026 06:31:05 -0700 Subject: [PATCH] Let WebRTC viewers mix a stream's audio stems When the latest segment lists stems (audio tracks in groups above the full mix's), the player menu gets an Audio section: the full mix by default, or each stem at Off/25/50/100%. WHEP offers three audio transceivers (full mix, then up to two stems). On web the video element plays the video and the full mix, and each stem mixed in plays from its own audio element; on native each track gets its own _setVolume. Track events from unused transceivers carry no stream and no longer replace the player's stream. Covered by flows/audio-stems.spec.ts and .maestro/logged-out/audio-stems.yaml, which run when the harness pushes over fMP4. --- .maestro/logged-out/audio-stems.yaml | 56 +++++++++++++ .../mobile-player/ui/viewer-context-menu.tsx | 78 +++++++++++++++++- .../components/mobile-player/use-webrtc.tsx | 34 ++++++-- .../mobile-player/video-async.native.tsx | 19 ++--- .../src/components/mobile-player/video.tsx | 80 +++++++++++++++++-- js/components/src/livestream-store/index.tsx | 1 + .../src/livestream-store/use-store.tsx | 13 ++- .../src/player-store/player-state.tsx | 6 ++ .../src/player-store/player-store.tsx | 19 ++++- js/core/src/index.ts | 1 + js/core/src/lib/audio-stems.test.ts | 80 +++++++++++++++++++ js/core/src/lib/audio-stems.ts | 46 +++++++++++ js/e2e-web/flows/audio-stems.spec.ts | 62 ++++++++++++++ 13 files changed, 470 insertions(+), 25 deletions(-) create mode 100644 .maestro/logged-out/audio-stems.yaml create mode 100644 js/core/src/lib/audio-stems.test.ts create mode 100644 js/core/src/lib/audio-stems.ts create mode 100644 js/e2e-web/flows/audio-stems.spec.ts diff --git a/.maestro/logged-out/audio-stems.yaml b/.maestro/logged-out/audio-stems.yaml new file mode 100644 index 000000000..8025af93f --- /dev/null +++ b/.maestro/logged-out/audio-stems.yaml @@ -0,0 +1,56 @@ +appId: ${APP_ID} +--- +# Over fMP4 ingest (SP_E2E_INGEST=fmp4) the harness pushes the test stream +# with a stem labeled "Music" beside the full mix. The player menu's Audio +# section lists it; picking a level mixes the stems in place of the full mix, +# and "Full mix" goes back. Web, which also checks what plays: +# js/e2e-web/flows/audio-stems.spec.ts. +- runFlow: + when: + true: ${typeof SP_E2E_INGEST !== "undefined" && SP_E2E_INGEST == "fmp4"} + commands: + - launchApp + # matched by testID because the card's link wrapper collapses its + # subtree on iOS + - extendedWaitUntil: + visible: + id: "home-stream-card" + timeout: 30000 + - tapOn: + id: "home-stream-card" + - waitForAnimationToEnd + - extendedWaitUntil: + visible: + text: ".*e2e test stream.*" + timeout: 30000 + # tap on the video to show the player controls + - tapOn: + point: 50%,20% + - tapOn: + id: "player-menu" + - extendedWaitUntil: + visible: + text: "Audio" + timeout: 10000 + - assertVisible: + id: "player-audio-stem-0" + # substring matches: iOS merges a menu row's texts into one element + - assertVisible: + text: ".*Music.*" + - takeScreenshot: audio-stems-menu + - tapOn: + id: "player-audio-stem-0" + - tapOn: + id: "player-audio-stem-0-level-50" + # the submenu's header, titled with the stem, leads back + - tapOn: + text: "Music" + - extendedWaitUntil: + visible: + text: ".*50%.*" + timeout: 10000 + - takeScreenshot: audio-stems-mixed + - tapOn: + id: "player-audio-full-mix" + - assertNotVisible: + text: ".*50%.*" diff --git a/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx b/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx index ea8b1398d..21d4329eb 100644 --- a/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx +++ b/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx @@ -7,7 +7,7 @@ import Animated, { withTiming, } from "react-native-reanimated"; import { useLivestreamInfo, useStreamplaceStore, zero } from "../../.."; -import { useLivestreamStore } from "../../../livestream-store"; +import { useAudioStems, useLivestreamStore } from "../../../livestream-store"; import { PlayerProtocol, usePlayerStore } from "../../../player-store/"; import { DropdownMenu, @@ -27,6 +27,11 @@ import { } from "../../ui"; import { ReportMenuItems } from "./report-menu-items"; +// The stem volumes a viewer can pick; 0 silences the stem. +const STEM_LEVELS = [0, 0.25, 0.5, 1]; +const stemLevelLabel = (level: number) => + level === 0 ? "Off" : `${level * 100}%`; + export function ContextMenu({ dropdownPortalContainer, onOpenChat, @@ -53,6 +58,12 @@ export function ContextMenu({ const isDevModeOn = useStreamplaceStore((x) => x.danmuUnlocked); + // Only WebRTC carries the stems (HLS has just the full mix), so the Audio + // section shows with low latency on. + const stems = useAudioStems(); + const stemVolumes = usePlayerStore((x) => x.stemVolumes); + const setStemVolumes = usePlayerStore((x) => x.setStemVolumes); + const latestSegment = useLivestreamStore((x) => x.segment); // get highest height x width rendition for video const videoRendition = latestSegment?.video?.reduce((prev, current) => { @@ -115,7 +126,7 @@ export function ContextMenu({ // rerender when dropdown portal container changes so we swap portals 'seamlessly' return ( - + @@ -202,6 +213,69 @@ export function ContextMenu({ + {mode !== "vod" && lowLatency && stems.length > 0 && ( + + + setStemVolumes(stemVolumes === null ? {} : null) + } + > + Full mix + + {stems.map((stem, i) => { + const level = stemVolumes?.[stem.alternateGroup] ?? 1; + return ( + + + + {stem.label} + {stemVolumes !== null && ( + + {stemLevelLabel(level)} + + )} + + + + + + setStemVolumes({ + ...stemVolumes, + [stem.alternateGroup]: Number(value), + }) + } + > + {STEM_LEVELS.map((l) => ( + + {stemLevelLabel(l)} + + ))} + + + + + ); + })} + + )} {onOpenChat && ( diff --git a/js/components/src/components/mobile-player/use-webrtc.tsx b/js/components/src/components/mobile-player/use-webrtc.tsx index 2f79b7bf0..5a4ae8c6c 100644 --- a/js/components/src/components/mobile-player/use-webrtc.tsx +++ b/js/components/src/components/mobile-player/use-webrtc.tsx @@ -15,8 +15,12 @@ import { RTCPeerConnection, RTCSessionDescription } from "./webrtc-primitives"; export default function useWebRTC( streamer: string, rendition: string, -): [MediaStream | null, boolean] { +): [MediaStream | null, boolean, (MediaStreamTrack | undefined)[]] { const [mediaStream, setMediaStream] = useState(null); + // The received audio tracks by transceiver: the full mix, then each stem. + const [audioTracks, setAudioTracks] = useState< + (MediaStreamTrack | undefined)[] + >([]); const [stuck, setStuck] = useState(false); const setStatus = usePlayerStore((x) => x.setStatus); let agent = usePossiblyUnauthedPDSAgent(); @@ -42,15 +46,33 @@ export default function useWebRTC( peerConnection.addTransceiver("video", { direction: "recvonly", }); - peerConnection.addTransceiver("audio", { - direction: "recvonly", - }); + // The node sends the full mix on the first audio transceiver and up to + // two stems on the others. + const audioTransceivers = [0, 1, 2].map(() => + peerConnection.addTransceiver("audio", { + direction: "recvonly", + }), + ); + const received: (MediaStreamTrack | undefined)[] = []; peerConnection.addEventListener("track", (event) => { const track = event.track; if (!track) { return; } - setMediaStream(event.streams[0]); + // An offered transceiver the node leaves unused still fires a track + // event, with no stream: keep the stream the real tracks arrived on. + if (event.streams[0]) { + setMediaStream(event.streams[0]); + } + const i = audioTransceivers.findIndex( + (t) => t.receiver.track?.id === track.id, + ); + if (i >= 0) { + received[i] = track; + } + // A fresh array for every track, so consumers that also read the + // stream's tracks catch up with each arrival. + setAudioTracks([...received]); }); peerConnection.addEventListener("connectionstatechange", () => { console.log("connection state change", peerConnection.connectionState); @@ -114,7 +136,7 @@ export default function useWebRTC( peerConnection.close(); }; }, [streamer, agent, isOwnStream, playbackWorkerUrl, rendition]); - return [mediaStream, stuck]; + return [mediaStream, stuck, audioTracks]; } /** diff --git a/js/components/src/components/mobile-player/video-async.native.tsx b/js/components/src/components/mobile-player/video-async.native.tsx index 1dc7419fa..8ca7d79bc 100644 --- a/js/components/src/components/mobile-player/video-async.native.tsx +++ b/js/components/src/components/mobile-player/video-async.native.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { LayoutChangeEvent, Linking } from "react-native"; import { MediaStream, + MediaStreamTrack, RTCView, RTCView as RTCViewIngest, } from "react-native-webrtc"; @@ -14,6 +15,7 @@ import { PlayerProtocol, PlayerStatus, Text, + useAudioTrackLevels, useEffectiveVolume, usePlayerStore as useIngestPlayerStore, useMuted, @@ -278,7 +280,7 @@ export function NativeWHEP(props?: { }) { const selectedRendition = usePlayerStore((x) => x.selectedRendition); const src = usePlayerStore((x) => x.src); - const [stream, stuck] = useWebRTC(src, selectedRendition); + const [stream, stuck, audioTracks] = useWebRTC(src, selectedRendition); const status = usePlayerStore((x) => x.status); const setPlayerWidth = usePlayerStore((x) => x.setPlayerWidth); @@ -303,6 +305,7 @@ export function NativeWHEP(props?: { const setStatus = usePlayerStore((x) => x.setStatus); const muted = useMuted(); const volume = useEffectiveVolume(); + const levels = useAudioTrackLevels(); useEffect(() => { if (stuck && status === PlayerStatus.PLAYING) { @@ -325,16 +328,14 @@ export function NativeWHEP(props?: { // setStatus(PlayerStatus.PLAYING); // }, [mediaStream, setStatus]); + // Every received audio track plays; the levels pick the full mix or stems. useEffect(() => { - if (!mediaStream) { - return; - } - mediaStream.getTracks().forEach((track) => { - if (track.kind === "audio") { - track._setVolume(muted ? 0 : volume); - } + audioTracks.forEach((track, i) => { + (track as unknown as MediaStreamTrack | undefined)?._setVolume( + muted ? 0 : volume * (levels[i] ?? 0), + ); }); - }, [mediaStream, muted, volume]); + }, [audioTracks, muted, volume, levels]); // Keep the playerStore videoRef in sync for PiP (if possible) useEffect(() => { diff --git a/js/components/src/components/mobile-player/video.tsx b/js/components/src/components/mobile-player/video.tsx index 04f12b7b2..a5c38c9fa 100644 --- a/js/components/src/components/mobile-player/video.tsx +++ b/js/components/src/components/mobile-player/video.tsx @@ -1,14 +1,23 @@ import type Hls from "hls.js"; -import { forwardRef, useCallback, useEffect, useRef, useState } from "react"; +import { + forwardRef, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { IngestMediaSource, PlayerProtocol, PlayerStatus, + useAudioTrackLevels, useEffectiveVolume, useMuted, usePlayerStore, useSetMuted, useStreamplaceStore, + useVolume, } from "../.."; import { borderRadius, mt } from "../../lib/theme/atoms"; import { statusColors, surfaces } from "../../lib/theme/tokens"; @@ -631,7 +640,28 @@ export function WebRTCPlayerInner({ const playerEvent = usePlayerStore((x) => x.playerEvent); const spurl = useStreamplaceStore((x) => x.url); - const [mediaStream, stuck] = useWebRTC(src, selectedRendition); + const [mediaStream, stuck, audioTracks] = useWebRTC(src, selectedRendition); + const levels = useAudioTrackLevels(); + const [fullMix, ...stems] = audioTracks; + + // The video element gets the video and the full mix; each stem the viewer + // mixes in plays from its own audio element below. A new array of audio + // tracks also means the stream has gained a track. + const videoStream = useMemo( + () => + mediaStream && + new MediaStream([ + ...mediaStream.getVideoTracks(), + ...(fullMix ? [fullMix] : []), + ]), + [mediaStream, audioTracks], + ); + + useEffect(() => { + if (fullMix) { + fullMix.enabled = levels[0] > 0; + } + }, [fullMix, levels]); useEffect(() => { if (stuck) { @@ -690,8 +720,8 @@ export function WebRTCPlayerInner({ if (!videoRef || !videoRef.current) { return; } - videoRef.current.srcObject = mediaStream; - }, [mediaStream]); + videoRef.current.srcObject = videoStream; + }, [videoStream]); if (!mediaStream) { const isError = connectionStatus === "connection-failed"; @@ -722,7 +752,47 @@ export function WebRTCPlayerInner({ ); } - return ; + return ( + <> + + {stems.map( + (track, i) => + track && + levels[i + 1] > 0 && ( + + ), + )} + + ); +} + +// One stem, playing at its own level under the player's volume and mute. +function StemAudio({ + track, + level, +}: { + track: MediaStreamTrack; + level: number; +}) { + const ref = useRef(null); + const volume = useVolume(); + const muted = useMuted(); + + useEffect(() => { + if (ref.current) { + ref.current.srcObject = new MediaStream([track]); + } + }, [track]); + + useEffect(() => { + if (ref.current) { + ref.current.volume = volume * level; + } + }, [volume, level]); + + return ( +