diff --git a/.maestro/logged-out/audio-stems.yaml b/.maestro/logged-out/audio-stems.yaml new file mode 100644 index 000000000..8025af93f --- /dev/null +++ b/.maestro/logged-out/audio-stems.yaml @@ -0,0 +1,56 @@ +appId: ${APP_ID} +--- +# Over fMP4 ingest (SP_E2E_INGEST=fmp4) the harness pushes the test stream +# with a stem labeled "Music" beside the full mix. The player menu's Audio +# section lists it; picking a level mixes the stems in place of the full mix, +# and "Full mix" goes back. Web, which also checks what plays: +# js/e2e-web/flows/audio-stems.spec.ts. +- runFlow: + when: + true: ${typeof SP_E2E_INGEST !== "undefined" && SP_E2E_INGEST == "fmp4"} + commands: + - launchApp + # matched by testID because the card's link wrapper collapses its + # subtree on iOS + - extendedWaitUntil: + visible: + id: "home-stream-card" + timeout: 30000 + - tapOn: + id: "home-stream-card" + - waitForAnimationToEnd + - extendedWaitUntil: + visible: + text: ".*e2e test stream.*" + timeout: 30000 + # tap on the video to show the player controls + - tapOn: + point: 50%,20% + - tapOn: + id: "player-menu" + - extendedWaitUntil: + visible: + text: "Audio" + timeout: 10000 + - assertVisible: + id: "player-audio-stem-0" + # substring matches: iOS merges a menu row's texts into one element + - assertVisible: + text: ".*Music.*" + - takeScreenshot: audio-stems-menu + - tapOn: + id: "player-audio-stem-0" + - tapOn: + id: "player-audio-stem-0-level-50" + # the submenu's header, titled with the stem, leads back + - tapOn: + text: "Music" + - extendedWaitUntil: + visible: + text: ".*50%.*" + timeout: 10000 + - takeScreenshot: audio-stems-mixed + - tapOn: + id: "player-audio-full-mix" + - assertNotVisible: + text: ".*50%.*" diff --git a/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx b/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx index ea8b1398d..21d4329eb 100644 --- a/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx +++ b/js/components/src/components/mobile-player/ui/viewer-context-menu.tsx @@ -7,7 +7,7 @@ import Animated, { withTiming, } from "react-native-reanimated"; import { useLivestreamInfo, useStreamplaceStore, zero } from "../../.."; -import { useLivestreamStore } from "../../../livestream-store"; +import { useAudioStems, useLivestreamStore } from "../../../livestream-store"; import { PlayerProtocol, usePlayerStore } from "../../../player-store/"; import { DropdownMenu, @@ -27,6 +27,11 @@ import { } from "../../ui"; import { ReportMenuItems } from "./report-menu-items"; +// The stem volumes a viewer can pick; 0 silences the stem. +const STEM_LEVELS = [0, 0.25, 0.5, 1]; +const stemLevelLabel = (level: number) => + level === 0 ? "Off" : `${level * 100}%`; + export function ContextMenu({ dropdownPortalContainer, onOpenChat, @@ -53,6 +58,12 @@ export function ContextMenu({ const isDevModeOn = useStreamplaceStore((x) => x.danmuUnlocked); + // Only WebRTC carries the stems (HLS has just the full mix), so the Audio + // section shows with low latency on. + const stems = useAudioStems(); + const stemVolumes = usePlayerStore((x) => x.stemVolumes); + const setStemVolumes = usePlayerStore((x) => x.setStemVolumes); + const latestSegment = useLivestreamStore((x) => x.segment); // get highest height x width rendition for video const videoRendition = latestSegment?.video?.reduce((prev, current) => { @@ -115,7 +126,7 @@ export function ContextMenu({ // rerender when dropdown portal container changes so we swap portals 'seamlessly' return ( - + @@ -202,6 +213,69 @@ export function ContextMenu({ + {mode !== "vod" && lowLatency && stems.length > 0 && ( + + + setStemVolumes(stemVolumes === null ? {} : null) + } + > + Full mix + + {stems.map((stem, i) => { + const level = stemVolumes?.[stem.alternateGroup] ?? 1; + return ( + + + + {stem.label} + {stemVolumes !== null && ( + + {stemLevelLabel(level)} + + )} + + + + + + setStemVolumes({ + ...stemVolumes, + [stem.alternateGroup]: Number(value), + }) + } + > + {STEM_LEVELS.map((l) => ( + + {stemLevelLabel(l)} + + ))} + + + + + ); + })} + + )} {onOpenChat && ( diff --git a/js/components/src/components/mobile-player/use-webrtc.tsx b/js/components/src/components/mobile-player/use-webrtc.tsx index 2f79b7bf0..5a4ae8c6c 100644 --- a/js/components/src/components/mobile-player/use-webrtc.tsx +++ b/js/components/src/components/mobile-player/use-webrtc.tsx @@ -15,8 +15,12 @@ import { RTCPeerConnection, RTCSessionDescription } from "./webrtc-primitives"; export default function useWebRTC( streamer: string, rendition: string, -): [MediaStream | null, boolean] { +): [MediaStream | null, boolean, (MediaStreamTrack | undefined)[]] { const [mediaStream, setMediaStream] = useState(null); + // The received audio tracks by transceiver: the full mix, then each stem. + const [audioTracks, setAudioTracks] = useState< + (MediaStreamTrack | undefined)[] + >([]); const [stuck, setStuck] = useState(false); const setStatus = usePlayerStore((x) => x.setStatus); let agent = usePossiblyUnauthedPDSAgent(); @@ -42,15 +46,33 @@ export default function useWebRTC( peerConnection.addTransceiver("video", { direction: "recvonly", }); - peerConnection.addTransceiver("audio", { - direction: "recvonly", - }); + // The node sends the full mix on the first audio transceiver and up to + // two stems on the others. + const audioTransceivers = [0, 1, 2].map(() => + peerConnection.addTransceiver("audio", { + direction: "recvonly", + }), + ); + const received: (MediaStreamTrack | undefined)[] = []; peerConnection.addEventListener("track", (event) => { const track = event.track; if (!track) { return; } - setMediaStream(event.streams[0]); + // An offered transceiver the node leaves unused still fires a track + // event, with no stream: keep the stream the real tracks arrived on. + if (event.streams[0]) { + setMediaStream(event.streams[0]); + } + const i = audioTransceivers.findIndex( + (t) => t.receiver.track?.id === track.id, + ); + if (i >= 0) { + received[i] = track; + } + // A fresh array for every track, so consumers that also read the + // stream's tracks catch up with each arrival. + setAudioTracks([...received]); }); peerConnection.addEventListener("connectionstatechange", () => { console.log("connection state change", peerConnection.connectionState); @@ -114,7 +136,7 @@ export default function useWebRTC( peerConnection.close(); }; }, [streamer, agent, isOwnStream, playbackWorkerUrl, rendition]); - return [mediaStream, stuck]; + return [mediaStream, stuck, audioTracks]; } /** diff --git a/js/components/src/components/mobile-player/video-async.native.tsx b/js/components/src/components/mobile-player/video-async.native.tsx index 1dc7419fa..8ca7d79bc 100644 --- a/js/components/src/components/mobile-player/video-async.native.tsx +++ b/js/components/src/components/mobile-player/video-async.native.tsx @@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { LayoutChangeEvent, Linking } from "react-native"; import { MediaStream, + MediaStreamTrack, RTCView, RTCView as RTCViewIngest, } from "react-native-webrtc"; @@ -14,6 +15,7 @@ import { PlayerProtocol, PlayerStatus, Text, + useAudioTrackLevels, useEffectiveVolume, usePlayerStore as useIngestPlayerStore, useMuted, @@ -278,7 +280,7 @@ export function NativeWHEP(props?: { }) { const selectedRendition = usePlayerStore((x) => x.selectedRendition); const src = usePlayerStore((x) => x.src); - const [stream, stuck] = useWebRTC(src, selectedRendition); + const [stream, stuck, audioTracks] = useWebRTC(src, selectedRendition); const status = usePlayerStore((x) => x.status); const setPlayerWidth = usePlayerStore((x) => x.setPlayerWidth); @@ -303,6 +305,7 @@ export function NativeWHEP(props?: { const setStatus = usePlayerStore((x) => x.setStatus); const muted = useMuted(); const volume = useEffectiveVolume(); + const levels = useAudioTrackLevels(); useEffect(() => { if (stuck && status === PlayerStatus.PLAYING) { @@ -325,16 +328,14 @@ export function NativeWHEP(props?: { // setStatus(PlayerStatus.PLAYING); // }, [mediaStream, setStatus]); + // Every received audio track plays; the levels pick the full mix or stems. useEffect(() => { - if (!mediaStream) { - return; - } - mediaStream.getTracks().forEach((track) => { - if (track.kind === "audio") { - track._setVolume(muted ? 0 : volume); - } + audioTracks.forEach((track, i) => { + (track as unknown as MediaStreamTrack | undefined)?._setVolume( + muted ? 0 : volume * (levels[i] ?? 0), + ); }); - }, [mediaStream, muted, volume]); + }, [audioTracks, muted, volume, levels]); // Keep the playerStore videoRef in sync for PiP (if possible) useEffect(() => { diff --git a/js/components/src/components/mobile-player/video.tsx b/js/components/src/components/mobile-player/video.tsx index 04f12b7b2..a5c38c9fa 100644 --- a/js/components/src/components/mobile-player/video.tsx +++ b/js/components/src/components/mobile-player/video.tsx @@ -1,14 +1,23 @@ import type Hls from "hls.js"; -import { forwardRef, useCallback, useEffect, useRef, useState } from "react"; +import { + forwardRef, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { IngestMediaSource, PlayerProtocol, PlayerStatus, + useAudioTrackLevels, useEffectiveVolume, useMuted, usePlayerStore, useSetMuted, useStreamplaceStore, + useVolume, } from "../.."; import { borderRadius, mt } from "../../lib/theme/atoms"; import { statusColors, surfaces } from "../../lib/theme/tokens"; @@ -631,7 +640,28 @@ export function WebRTCPlayerInner({ const playerEvent = usePlayerStore((x) => x.playerEvent); const spurl = useStreamplaceStore((x) => x.url); - const [mediaStream, stuck] = useWebRTC(src, selectedRendition); + const [mediaStream, stuck, audioTracks] = useWebRTC(src, selectedRendition); + const levels = useAudioTrackLevels(); + const [fullMix, ...stems] = audioTracks; + + // The video element gets the video and the full mix; each stem the viewer + // mixes in plays from its own audio element below. A new array of audio + // tracks also means the stream has gained a track. + const videoStream = useMemo( + () => + mediaStream && + new MediaStream([ + ...mediaStream.getVideoTracks(), + ...(fullMix ? [fullMix] : []), + ]), + [mediaStream, audioTracks], + ); + + useEffect(() => { + if (fullMix) { + fullMix.enabled = levels[0] > 0; + } + }, [fullMix, levels]); useEffect(() => { if (stuck) { @@ -690,8 +720,8 @@ export function WebRTCPlayerInner({ if (!videoRef || !videoRef.current) { return; } - videoRef.current.srcObject = mediaStream; - }, [mediaStream]); + videoRef.current.srcObject = videoStream; + }, [videoStream]); if (!mediaStream) { const isError = connectionStatus === "connection-failed"; @@ -722,7 +752,47 @@ export function WebRTCPlayerInner({ ); } - return ; + return ( + <> + + {stems.map( + (track, i) => + track && + levels[i + 1] > 0 && ( + + ), + )} + + ); +} + +// One stem, playing at its own level under the player's volume and mute. +function StemAudio({ + track, + level, +}: { + track: MediaStreamTrack; + level: number; +}) { + const ref = useRef(null); + const volume = useVolume(); + const muted = useMuted(); + + useEffect(() => { + if (ref.current) { + ref.current.srcObject = new MediaStream([track]); + } + }, [track]); + + useEffect(() => { + if (ref.current) { + ref.current.volume = volume * level; + } + }, [volume, level]); + + return ( +