From 61a7ed223b1c9f9d966ce637c0b28e8d76b3e20a Mon Sep 17 00:00:00 2001 From: Seth Call Date: Mon, 2 Mar 2026 08:34:45 -0600 Subject: [PATCH] stable stopping point with initial webrtc attempts --- .../async-bridge-spike/progress.md | 5 + agent-tasks/client-simulator/progress.md | 11 +- web/app/assets/javascripts/application.js | 1 + .../application_client_modern.js | 1 + web/app/assets/javascripts/homeScreen.js | 61 ++- web/app/assets/javascripts/jam_rest.js | 6 + .../actions/SessionActions.js.coffee | 26 + .../mixins/SessionMyTracksMixin.js.coffee | 5 +- .../stores/ConfigureTracksStore.js.coffee | 14 +- .../stores/SessionStore.js.coffee | 6 + .../stores/SessionStoreModern.es6 | 34 ++ web/app/assets/javascripts/sessionModel.js | 7 + web/app/assets/javascripts/webJamClient.js | 483 ++++++++++++++++++ web/app/helpers/client_helper.rb | 2 + web/package.json | 16 +- 15 files changed, 652 insertions(+), 26 deletions(-) diff --git a/agent-tasks/client-simulator/async-bridge-spike/progress.md b/agent-tasks/client-simulator/async-bridge-spike/progress.md index 66f01f68d..9ec50425c 100644 --- a/agent-tasks/client-simulator/async-bridge-spike/progress.md +++ b/agent-tasks/client-simulator/async-bridge-spike/progress.md @@ -232,3 +232,8 @@ Determine whether frontend code can support `await context.jamClient.method()` f - `session_utils.js` join-source trace now writes to buffer (`join-source.session-utils.joinSession`) while downgrading console severity. - Added a hard gate in `SessionStoreModern.es6` so the modern Reflux store only installs when `window.__JK_SKIP_LEGACY_SESSION_STORE__` is true. - In legacy bundle loads, modern store now emits `join-source.session-store.modern-skipped` and does not register listeners; this prevents hidden second `onJoinSession` listeners from racing the legacy store. +- Added targeted leave-source instrumentation to diagnose immediate post-join session departure: +- `SessionActions.leaveSession` now emits `leave-source.session-action` (call + trigger wrappers, with stack) in `react-components/actions/SessionActions.js.coffee`. +- `SessionStore.js.coffee#onLeaveSession` now emits `leave-source.session-store.legacy-onLeaveSession`. +- `SessionStoreModern.es6#onLeaveSession` now emits `leave-source.session-store.modern-onLeaveSession`. +- This is intended to identify the exact caller path behind immediate `DELETE /api/participants/:client_id` after successful join. diff --git a/agent-tasks/client-simulator/progress.md b/agent-tasks/client-simulator/progress.md index 4de611fde..33186da83 100644 --- a/agent-tasks/client-simulator/progress.md +++ b/agent-tasks/client-simulator/progress.md @@ -7,7 +7,7 @@ Build a web-client-backed `jamClient` compatibility path plus a simulator/test h - [x] Create task tracking and record the initial plan - [ ] Complete async bridge contract spike (`await context.jamClient.method()` viability and blast radius) - [ ] Complete async bridge contract spike (`await context.jamClient.method()` viability and blast radius) (dual-bundle parser split + `SessionStore` legacy/modern split in progress) -- [ ] Map protocol surface (jamClient methods, backend callbacks, REST APIs, websocket messages, shared-object usage) +- [x] Map protocol surface for join/mute/volume flows using native 2-party recordings - [ ] Add instrumentation/tape-recording hooks (frontend/backend) - [ ] Define comparison assertions for compatibility tests - [ ] Implement web `jamClient` shim compatibility layer @@ -15,6 +15,15 @@ Build a web-client-backed `jamClient` compatibility path plus a simulator/test h - [ ] Build MVP web-to-web session flow - [ ] Build simulator/replay harness and automated specs +## 2026-03-01 WebRTC Planning Update +- Captured native reference logs for 2-party mute/volume scenario in: + - `web/ai/native-client-2p-recording/20260228-210957-seth-native-2p-mute-volume.log` + - `web/ai/native-client-2p-recording/20260228-211006-david-native-2p-mute-volume.log` +- Confirmed these are sufficient as baseline artifacts for bridge-call and REST/websocket pattern parity while implementing web-client media with selective behavioral divergence. +- Added execution plan in `agent-tasks/client-simulator/webrtc-web-client-plan/progress.md`. +- Added Tier A/B/C contract matrix from the captured logs in: + - `web/ai/native-client-2p-recording/contract-matrix.md` + ## Notes - Native recording execution is intentionally out of scope for this effort (native path is being deprecated). - Recording/comparison scope must include REST API requests/responses in addition to bridge/events/websocket/DB behavior. diff --git a/web/app/assets/javascripts/application.js b/web/app/assets/javascripts/application.js index ff1bb12e6..1fa9e3552 100644 --- a/web/app/assets/javascripts/application.js +++ b/web/app/assets/javascripts/application.js @@ -54,6 +54,7 @@ //= require AAA_Log //= require globals //= require debug_log_collector +//= require mode_flags_overlay //= require AAB_message_factory //= require jam_rest //= require ga diff --git a/web/app/assets/javascripts/client_bundles/application_client_modern.js b/web/app/assets/javascripts/client_bundles/application_client_modern.js index 553278421..12a681e46 100644 --- a/web/app/assets/javascripts/client_bundles/application_client_modern.js +++ b/web/app/assets/javascripts/client_bundles/application_client_modern.js @@ -54,6 +54,7 @@ //= require AAA_Log //= require globals //= require debug_log_collector +//= require mode_flags_overlay //= require AAB_message_factory //= require jam_rest //= require ga diff --git a/web/app/assets/javascripts/homeScreen.js b/web/app/assets/javascripts/homeScreen.js index cf4cd9372..0e1d846e8 100644 --- a/web/app/assets/javascripts/homeScreen.js +++ b/web/app/assets/javascripts/homeScreen.js @@ -28,21 +28,56 @@ $(this).removeClass('hover'); } - function switchClientMode(e) { - // ctrl + shift + 0 - if(e.ctrlKey && e.shiftKey && e.keyCode == 48) { - logger.debug("switch client mode!"); - var act_as_native_client = $.cookie('act_as_native_client'); - - logger.debug("currently: " + act_as_native_client); - if(act_as_native_client == null || act_as_native_client != "true") { - logger.debug("forcing act as native client!"); - $.cookie('act_as_native_client', 'true', { expires: 120, path: '/' }); + function setWebClientWebRtcFlag(enabled) { + try { + if (!window.localStorage) { return; } + if (enabled) { + window.localStorage.setItem('jk.webClient.webrtc', '1'); } else { - logger.debug("remove act as native client!"); - $.removeCookie('act_as_native_client'); + window.localStorage.removeItem('jk.webClient.webrtc'); } + } + catch(err) { + logger.warn("unable to update jk.webClient.webrtc localStorage flag", err); + } + } + + function toggleActAsNativeClient() { + var act_as_native_client = $.cookie('act_as_native_client'); + var enable = (act_as_native_client == null || act_as_native_client != "true"); + + if(enable) { + logger.debug("forcing act as native client!"); + $.cookie('act_as_native_client', 'true', { expires: 120, path: '/' }); + } + else { + logger.debug("remove act as native client!"); + $.removeCookie('act_as_native_client'); + } + + return enable; + } + + function switchClientMode(e) { + if(!(e.ctrlKey && e.shiftKey)) { return; } + + // ctrl + shift + 0 + if(e.keyCode == 48) { + logger.debug("switch client mode!"); + toggleActAsNativeClient(); + window.location.reload(); + return; + } + + // ctrl + shift + 9 + // One-step toggle for web client mode: + // - act_as_native_client cookie + // - localStorage flag enabling WebRTC path in WebJamClient + if(e.keyCode == 57) { + logger.debug("switch web client mode (native-cookie + webrtc flag)!"); + var enabled = toggleActAsNativeClient(); + setWebClientWebRtcFlag(enabled); window.location.reload(); } } @@ -128,4 +163,4 @@ this.beforeShow = beforeShow; }; - })(window,jQuery); \ No newline at end of file + })(window,jQuery); diff --git a/web/app/assets/javascripts/jam_rest.js b/web/app/assets/javascripts/jam_rest.js index c06a236d6..753b0eb94 100644 --- a/web/app/assets/javascripts/jam_rest.js +++ b/web/app/assets/javascripts/jam_rest.js @@ -594,6 +594,12 @@ } function deleteParticipant(clientId) { + if (context.JK && context.JK.DebugLogCollector && context.JK.DebugLogCollector.push) { + context.JK.DebugLogCollector.push("leave-source.rest.deleteParticipant", { + client_id: clientId, + stack: (new Error("jam_rest.deleteParticipant")).stack + }); + } var url = "/api/participants/" + clientId; return $.ajax({ type: "DELETE", diff --git a/web/app/assets/javascripts/react-components/actions/SessionActions.js.coffee b/web/app/assets/javascripts/react-components/actions/SessionActions.js.coffee index 1ef85945c..62d3b18ef 100644 --- a/web/app/assets/javascripts/react-components/actions/SessionActions.js.coffee +++ b/web/app/assets/javascripts/react-components/actions/SessionActions.js.coffee @@ -51,3 +51,29 @@ if @SessionActions?.joinSession? originalTrigger.apply(originalJoinAction, arguments) @SessionActions.joinSession = wrappedJoinAction + +pushLeaveActionTrace = (source, argsLike) -> + return unless context.JK?.DebugLogCollector?.push + args = Array::slice.call(argsLike ? []) + context.JK.DebugLogCollector.push("leave-source.session-action", { + source: source + args: args + stack: (new Error("SessionActions.leaveSession")).stack + }) + +if @SessionActions?.leaveSession? + originalLeaveAction = @SessionActions.leaveSession + wrappedLeaveAction = -> + pushLeaveActionTrace('call', arguments) + originalLeaveAction.apply(this, arguments) + + for own key, value of originalLeaveAction + wrappedLeaveAction[key] = value + + if originalLeaveAction.trigger? + originalLeaveTrigger = originalLeaveAction.trigger + wrappedLeaveAction.trigger = -> + pushLeaveActionTrace('trigger', arguments) + originalLeaveTrigger.apply(originalLeaveAction, arguments) + + @SessionActions.leaveSession = wrappedLeaveAction diff --git a/web/app/assets/javascripts/react-components/mixins/SessionMyTracksMixin.js.coffee b/web/app/assets/javascripts/react-components/mixins/SessionMyTracksMixin.js.coffee index 87fe9f0fb..0910f98b2 100644 --- a/web/app/assets/javascripts/react-components/mixins/SessionMyTracksMixin.js.coffee +++ b/web/app/assets/javascripts/react-components/mixins/SessionMyTracksMixin.js.coffee @@ -65,6 +65,7 @@ MIDI_TRACK = context.JK.MIDI_TRACK # find any VST info if hasMixer && @configureTracks? + vstAssignments = @configureTracks?.vstTrackAssignments?.vsts ? [] # bug in the backend; track is wrong for personal mixers (always 1), but correct for master mix trackAssignment = -1 @@ -79,7 +80,7 @@ MIDI_TRACK = context.JK.MIDI_TRACK else trackAssignment = mixerData.oppositeMixer?.track - for vst in @configureTracks.vstTrackAssignments.vsts + for vst in vstAssignments if vst.track == trackAssignment - 1 && vst.name != 'NONE' logger.debug("found VST on track", vst, track) associatedVst = vst @@ -92,4 +93,4 @@ MIDI_TRACK = context.JK.MIDI_TRACK this.setState(tracks: tracks, mySession:session, chat: chat) -} \ No newline at end of file +} diff --git a/web/app/assets/javascripts/react-components/stores/ConfigureTracksStore.js.coffee b/web/app/assets/javascripts/react-components/stores/ConfigureTracksStore.js.coffee index 8a04b9c68..4bbade589 100644 --- a/web/app/assets/javascripts/react-components/stores/ConfigureTracksStore.js.coffee +++ b/web/app/assets/javascripts/react-components/stores/ConfigureTracksStore.js.coffee @@ -242,6 +242,8 @@ void removeSearchPath(int typeId, QString pathToRemove); # the backend does not have a consistent way of tracking assigned inputs for midi. # let's make it seem consistent injectMidiToTrackAssignments: () -> + return unless @trackAssignments?.inputs?.assigned? + return unless @vstTrackAssignments?.vsts? if @vstTrackAssignments? for vst in @vstTrackAssignments.vsts if vst.track == MIDI_TRACK - 1 @@ -263,26 +265,28 @@ void removeSearchPath(int typeId, QString pathToRemove); changed: () -> @injectMidiToTrackAssignments() + assignedInputs = @trackAssignments?.inputs?.assigned ? [] + vstAssignments = @vstTrackAssignments?.vsts ? [] @editingTrack = [] @editingTrack.assignment = @trackNumber if @trackNumber? - for inputsForTrack in @trackAssignments.inputs.assigned + for inputsForTrack in assignedInputs if inputsForTrack.assignment == @trackNumber @editingTrack = inputsForTrack break # slap on vst, if any, from list of vst assignments - for vst in @vstTrackAssignments.vsts + for vst in vstAssignments if vst.track == @editingTrack.assignment - 1 @editingTrack.vst = vst @editingTrack.midiDeviceIndex = vst.midiDeviceIndex break - for inputsForTrack in @trackAssignments.inputs.assigned + for inputsForTrack in assignedInputs if vst.track == inputsForTrack.assignment - 1 inputsForTrack.vst = vst @@ -290,7 +294,7 @@ void removeSearchPath(int typeId, QString pathToRemove); logger.debug("current track has a VST assigned:" + @editingTrack.vst.file) - unscanned = !@scannedBefore && @vstPluginList.vsts.length <= 1 + unscanned = !@scannedBefore && ((@vstPluginList?.vsts?.length ? 0) <= 1) @item = { unscanned: unscanned, @@ -576,4 +580,4 @@ void removeSearchPath(int typeId, QString pathToRemove); )), 250) } -) \ No newline at end of file +) diff --git a/web/app/assets/javascripts/react-components/stores/SessionStore.js.coffee b/web/app/assets/javascripts/react-components/stores/SessionStore.js.coffee index 047113221..a58771343 100644 --- a/web/app/assets/javascripts/react-components/stores/SessionStore.js.coffee +++ b/web/app/assets/javascripts/react-components/stores/SessionStore.js.coffee @@ -1169,6 +1169,12 @@ if context.JK?.DebugLogCollector?.push # called by anyone wanting to leave the session with a certain behavior onLeaveSession: (behavior) -> logger.debug("attempting to leave session", behavior) + if context.JK?.DebugLogCollector?.push + context.JK.DebugLogCollector.push("leave-source.session-store.legacy-onLeaveSession", { + behavior: behavior + current_session_id: @currentSessionId + stack: (new Error("SessionStoreLegacy.onLeaveSession")).stack + }) if behavior.notify @app.layout.notify(behavior.notify) diff --git a/web/app/assets/javascripts/react-components/stores/SessionStoreModern.es6 b/web/app/assets/javascripts/react-components/stores/SessionStoreModern.es6 index 27d35d913..5b8a5841e 100644 --- a/web/app/assets/javascripts/react-components/stores/SessionStoreModern.es6 +++ b/web/app/assets/javascripts/react-components/stores/SessionStoreModern.es6 @@ -809,6 +809,23 @@ if (shouldInstallModernSessionStore) { }, async onJoinSessionDone(musicSession) { + const pushJoinAbort = (reason, detail) => { + context.__jkJoinAborts = context.__jkJoinAborts || []; + context.__jkJoinAborts.push({ + ts: (new Date()).toISOString(), + reason, + detail, + current_session_id: this.currentSessionId + }); + if (context.JK && context.JK.DebugLogCollector && context.JK.DebugLogCollector.push) { + context.JK.DebugLogCollector.push("join-source.session-store.modern-abort", { + reason, + detail, + current_session_id: this.currentSessionId, + stack: (new Error("SessionStoreModern.onJoinSessionDone.abort")).stack + }); + } + }; const musicianAccessOnJoin = musicSession.musician_access; const shouldVerifyNetwork = musicSession.musician_access; @@ -827,6 +844,7 @@ if (shouldInstallModernSessionStore) { try { await this.gearUtils.guardAgainstInvalidConfiguration(this.app, shouldVerifyNetwork); } catch (e) { + pushJoinAbort("guardAgainstInvalidConfiguration", {error: String(e)}); SessionActions.leaveSession.trigger({location: '/client#/home'}); return; } @@ -848,6 +866,7 @@ if (shouldInstallModernSessionStore) { leaveBehavior.location = '/client#/home'; } + pushJoinAbort("guardAgainstActiveProfileMissing", data); SessionActions.leaveSession.trigger(leaveBehavior); return; } @@ -864,6 +883,7 @@ if (shouldInstallModernSessionStore) { context.JK.alertSupportedNeeded('Unable to determine configured tracks due to reason: ' + data); } + pushJoinAbort("waitForSessionPageEnterDone", {error: data}); SessionActions.leaveSession.trigger({location: '/client#/home'}); return; } @@ -1250,6 +1270,13 @@ if (shouldInstallModernSessionStore) { leaveSessionRest() { + if (context.JK && context.JK.DebugLogCollector && context.JK.DebugLogCollector.push) { + context.JK.DebugLogCollector.push("leave-source.session-store.modern-leaveSessionRest", { + current_session_id: this.currentSessionId, + client_id: this.app && this.app.clientId, + stack: (new Error("SessionStoreModern.leaveSessionRest")).stack + }); + } return rest.deleteParticipant(this.app.clientId); }, @@ -1436,6 +1463,13 @@ if (shouldInstallModernSessionStore) { // called by anyone wanting to leave the session with a certain behavior onLeaveSession(behavior) { logger.debug("attempting to leave session", behavior); + if (context.JK && context.JK.DebugLogCollector && context.JK.DebugLogCollector.push) { + context.JK.DebugLogCollector.push("leave-source.session-store.modern-onLeaveSession", { + behavior: behavior, + current_session_id: this.currentSessionId, + stack: (new Error("SessionStoreModern.onLeaveSession")).stack + }); + } if (behavior.notify) { this.app.layout.notify(behavior.notify); diff --git a/web/app/assets/javascripts/sessionModel.js b/web/app/assets/javascripts/sessionModel.js index db32af567..6f44493b5 100644 --- a/web/app/assets/javascripts/sessionModel.js +++ b/web/app/assets/javascripts/sessionModel.js @@ -631,6 +631,13 @@ } function leaveSessionRest(sessionId) { + if (context.JK && context.JK.DebugLogCollector && context.JK.DebugLogCollector.push) { + context.JK.DebugLogCollector.push("leave-source.session-model.leaveSessionRest", { + session_id: sessionId, + client_id: clientId, + stack: (new Error("SessionModel.leaveSessionRest")).stack + }); + } var url = "/api/participants/" + clientId; return $.ajax({ type: "DELETE", diff --git a/web/app/assets/javascripts/webJamClient.js b/web/app/assets/javascripts/webJamClient.js index 345e48c2e..6833e02ca 100644 --- a/web/app/assets/javascripts/webJamClient.js +++ b/web/app/assets/javascripts/webJamClient.js @@ -29,6 +29,20 @@ stream: null, error: null, autoStartEnabled: false + }, + controlState: { + master: {}, + personal: {}, + initialized: { + master: false, + personal: false + } + }, + participants: {}, + webrtc: { + enabled: false, + peerConnections: {}, + localClientId: null } }; @@ -228,6 +242,352 @@ pushRecord({kind: "webClientMediaStatus", status: state.localMedia.status}); } + function ensureLocalMediaForWebRtc() { + if (!state.webrtc.enabled) { return; } + if (!context.navigator || !context.navigator.mediaDevices || !context.navigator.mediaDevices.getUserMedia) { + return; + } + if (state.localMedia.status === "requesting" || state.localMedia.status === "active") { + return; + } + + state.localMedia.status = "requesting"; + state.localMedia.error = null; + pushRecord({kind: "webClientMediaStatus", status: state.localMedia.status, reason: "webrtc"}); + + context.navigator.mediaDevices.getUserMedia({audio: true, video: false}) + .then(function(stream) { + state.localMedia.stream = stream; + state.localMedia.status = "active"; + state.localMedia.error = null; + pushRecord({ + kind: "webClientMediaStatus", + status: state.localMedia.status, + reason: "webrtc", + tracks: stream && stream.getAudioTracks ? stream.getAudioTracks().length : 0 + }); + + Object.keys(state.webrtc.peerConnections).forEach(function(clientId) { + var entry = state.webrtc.peerConnections[clientId]; + if (entry && entry.pc) { + addLocalTracksToPeerConnection(entry.pc); + } + }); + }) + .catch(function(err) { + state.localMedia.stream = null; + state.localMedia.status = "failed"; + state.localMedia.error = (err && err.name) ? err.name : String(err); + pushRecord({ + kind: "webClientMediaStatus", + status: state.localMedia.status, + reason: "webrtc", + error: state.localMedia.error + }); + }); + } + + function modeKey(isMasterOrPersonal) { + return isMasterOrPersonal ? "master" : "personal"; + } + + function deepClone(value) { + return safeSerialize(value); + } + + function getJamServer() { + return context.JK && context.JK.JamServer ? context.JK.JamServer : null; + } + + function getSelfClientId() { + if (state.webrtc.localClientId) { return state.webrtc.localClientId; } + if (state.app && state.app.clientId) { return state.app.clientId; } + var server = getJamServer(); + if (server && server.clientID) { return server.clientID; } + return null; + } + + function readWebRtcEnabledFlag() { + return !!context.JK_WEB_CLIENT_WEBRTC_ENABLED || + readLocalStorageFlag("jk.webClient.webrtc") || + !!(context.gon && context.gon.web_client_webrtc_enabled); + } + + function ensureControlStateCache(isMasterOrPersonal) { + var key = modeKey(isMasterOrPersonal); + if (state.controlState.initialized[key]) { + return; + } + + var delegateState = invokeDelegate("SessionGetAllControlState", [isMasterOrPersonal]); + var byId = {}; + if (delegateState && delegateState.length) { + delegateState.forEach(function(mixer) { + if (!mixer || !mixer.id) { return; } + byId[mixer.id] = deepClone(mixer); + }); + } + + state.controlState[key] = byId; + state.controlState.initialized[key] = true; + } + + var syncTracksTimer = null; + function scheduleTrackSync() { + if (!context.MixerActions || !context.MixerActions.syncTracks) { return; } + if (syncTracksTimer) { + context.clearTimeout(syncTracksTimer); + } + syncTracksTimer = context.setTimeout(function() { + syncTracksTimer = null; + try { + context.MixerActions.syncTracks(); + pushRecord({kind: "webClientTrackSync", source: "SessionSetControlState"}); + } catch (e) { + pushRecord({ + kind: "webClientTrackSyncError", + error: (e && e.message) ? e.message : String(e) + }); + } + }, 50); + } + + function applyTrackVolumeToCache(mixerId, isMasterOrPersonal) { + ensureControlStateCache(isMasterOrPersonal); + var key = modeKey(isMasterOrPersonal); + var cache = state.controlState[key]; + var existing = cache[mixerId] || {}; + var trackVolume = context.trackVolumeObject || {}; + + var updated = deepClone(existing) || {}; + updated.id = mixerId; + updated.master = !!isMasterOrPersonal; + updated.monitor = !isMasterOrPersonal; + if (trackVolume.clientID !== undefined) { updated.client_id = trackVolume.clientID; } + if (trackVolume.mute !== undefined) { updated.mute = !!trackVolume.mute; } + if (trackVolume.volL !== undefined) { updated.volume_left = trackVolume.volL; } + if (trackVolume.volR !== undefined) { updated.volume_right = trackVolume.volR; } + if (trackVolume.pan !== undefined) { updated.pan = trackVolume.pan; } + if (trackVolume.loop !== undefined) { updated.loop = !!trackVolume.loop; } + if (trackVolume.name !== undefined) { updated.name = trackVolume.name; } + if (trackVolume.record !== undefined) { updated.record = !!trackVolume.record; } + if (updated.range_low === undefined) { updated.range_low = -80; } + if (updated.range_high === undefined) { updated.range_high = 20; } + + cache[mixerId] = updated; + return updated; + } + + function updateLocalTrackMuteFromControlState(controlState) { + if (!controlState) { return; } + if (!state.localMedia.stream || !state.localMedia.stream.getAudioTracks) { return; } + + var trackClientId = controlState.client_id || ""; + var selfClientId = getSelfClientId(); + var isSelfControl = !trackClientId || (selfClientId && trackClientId === selfClientId); + if (!isSelfControl) { return; } + + var shouldMute = !!controlState.mute; + state.localMedia.stream.getAudioTracks().forEach(function(track) { + try { + track.enabled = !shouldMute; + } catch (e) {} + }); + } + + function getPeerConnectionCtor() { + return context.RTCPeerConnection || context.webkitRTCPeerConnection || context.mozRTCPeerConnection; + } + + function sendWebRtcSignal(receiverClientId, payload) { + var server = getJamServer(); + if (!server || typeof server.sendP2PMessage !== "function") { + pushRecord({kind: "webrtc.signalDropped", reason: "jamServerUnavailable", to: receiverClientId}); + return; + } + + var envelope = { + jk_webclient_webrtc: true, + session_id: state.currentSessionId, + from_client_id: getSelfClientId(), + payload: payload + }; + + try { + server.sendP2PMessage(receiverClientId, JSON.stringify(envelope)); + pushRecord({ + kind: "webrtc.signalSent", + to: receiverClientId, + signal_type: payload && payload.type ? payload.type : "candidate" + }); + } catch (e) { + pushRecord({ + kind: "webrtc.signalSendError", + to: receiverClientId, + error: (e && e.message) ? e.message : String(e) + }); + } + } + + function addLocalTracksToPeerConnection(peerConnection) { + if (!peerConnection || !state.localMedia.stream || !state.localMedia.stream.getTracks) { + return; + } + + var stream = state.localMedia.stream; + stream.getTracks().forEach(function(track) { + try { + peerConnection.addTrack(track, stream); + } catch (e) {} + }); + } + + function closePeerConnection(clientId) { + var entry = state.webrtc.peerConnections[clientId]; + if (!entry) { return; } + try { + if (entry.pc && typeof entry.pc.close === "function") { + entry.pc.close(); + } + } catch (e) {} + delete state.webrtc.peerConnections[clientId]; + pushRecord({kind: "webrtc.peerClosed", client_id: clientId}); + } + + function ensurePeerConnection(clientId) { + if (!state.webrtc.enabled || !clientId) { return null; } + if (state.webrtc.peerConnections[clientId]) { + return state.webrtc.peerConnections[clientId]; + } + + var RTCPeerConnectionCtor = getPeerConnectionCtor(); + if (!RTCPeerConnectionCtor) { + pushRecord({kind: "webrtc.unavailable", reason: "RTCPeerConnection"}); + return null; + } + + var iceServers = (context.gon && context.gon.web_client_ice_servers) || []; + var peerConnection = null; + try { + peerConnection = new RTCPeerConnectionCtor({iceServers: iceServers}); + } catch (e) { + pushRecord({ + kind: "webrtc.peerCreateError", + client_id: clientId, + error: (e && e.message) ? e.message : String(e) + }); + return null; + } + + var entry = { + client_id: clientId, + pc: peerConnection + }; + state.webrtc.peerConnections[clientId] = entry; + + addLocalTracksToPeerConnection(peerConnection); + + peerConnection.onicecandidate = function(evt) { + if (!evt || !evt.candidate) { return; } + sendWebRtcSignal(clientId, { + type: "candidate", + candidate: evt.candidate + }); + }; + + peerConnection.onconnectionstatechange = function() { + pushRecord({ + kind: "webrtc.connectionState", + client_id: clientId, + state: peerConnection.connectionState + }); + }; + + peerConnection.ontrack = function(evt) { + entry.remoteStream = evt && evt.streams && evt.streams.length ? evt.streams[0] : null; + pushRecord({ + kind: "webrtc.remoteTrack", + client_id: clientId, + has_stream: !!entry.remoteStream + }); + }; + + pushRecord({kind: "webrtc.peerCreated", client_id: clientId}); + return entry; + } + + function maybeCreateOffer(clientId) { + var entry = ensurePeerConnection(clientId); + if (!entry || !entry.pc || typeof entry.pc.createOffer !== "function") { return; } + + entry.pc.createOffer() + .then(function(offer) { + return entry.pc.setLocalDescription(offer).then(function() { + sendWebRtcSignal(clientId, {type: "offer", sdp: offer}); + }); + }) + .catch(function(err) { + pushRecord({ + kind: "webrtc.offerError", + client_id: clientId, + error: (err && err.message) ? err.message : String(err) + }); + }); + } + + function onWebRtcSignal(fromClientId, signalPayload) { + if (!state.webrtc.enabled || !signalPayload || !fromClientId) { return false; } + var entry = ensurePeerConnection(fromClientId); + if (!entry || !entry.pc) { return true; } + + var pc = entry.pc; + if (signalPayload.type === "offer") { + pc.setRemoteDescription(signalPayload.sdp) + .then(function() { + return pc.createAnswer(); + }) + .then(function(answer) { + return pc.setLocalDescription(answer).then(function() { + sendWebRtcSignal(fromClientId, {type: "answer", sdp: answer}); + }); + }) + .catch(function(err) { + pushRecord({ + kind: "webrtc.answerError", + client_id: fromClientId, + error: (err && err.message) ? err.message : String(err) + }); + }); + return true; + } + + if (signalPayload.type === "answer") { + pc.setRemoteDescription(signalPayload.sdp).catch(function(err) { + pushRecord({ + kind: "webrtc.setAnswerError", + client_id: fromClientId, + error: (err && err.message) ? err.message : String(err) + }); + }); + return true; + } + + if (signalPayload.type === "candidate") { + if (signalPayload.candidate && typeof pc.addIceCandidate === "function") { + pc.addIceCandidate(signalPayload.candidate).catch(function(err) { + pushRecord({ + kind: "webrtc.addCandidateError", + client_id: fromClientId, + error: (err && err.message) ? err.message : String(err) + }); + }); + } + return true; + } + + return false; + } + function installInstrumentationWrappers(target) { Object.keys(target).forEach(function(key) { var original = target[key]; @@ -309,6 +669,11 @@ error: state.localMedia.error, hasStream: !!state.localMedia.stream, autoStartEnabled: !!state.localMedia.autoStartEnabled + }, + webrtc: { + enabled: !!state.webrtc.enabled, + localClientId: getSelfClientId(), + peerCount: Object.keys(state.webrtc.peerConnections).length } }, recorder: { @@ -384,6 +749,9 @@ } pushRecord({kind: "webClientSessionState", event: "JoinSession", session: safeSerialize(sessionDescriptor)}); startLocalMediaIfEnabled(); + if (state.webrtc.enabled) { + ensureLocalMediaForWebRtc(); + } return invokeDelegate("JoinSession", arguments); }; @@ -391,17 +759,132 @@ state.inSession = false; state.currentSessionId = null; pushRecord({kind: "webClientSessionState", event: "LeaveSession", session: safeSerialize(sessionDescriptor)}); + Object.keys(state.webrtc.peerConnections).forEach(function(clientId) { + closePeerConnection(clientId); + }); if (!state.inSessionPage) { stopLocalMedia(); } return invokeDelegate("LeaveSession", arguments); }; + this.SessionGetAllControlState = function(isMasterOrPersonal) { + ensureControlStateCache(isMasterOrPersonal); + var key = modeKey(isMasterOrPersonal); + var values = Object.keys(state.controlState[key]).map(function(mixerId) { + return deepClone(state.controlState[key][mixerId]); + }); + pushRecord({ + kind: "webClientControlStateRead", + mode: key, + count: values.length + }); + return values; + }; + + this.SessionSetControlState = function(mixerId, isMasterOrPersonal) { + var updated = applyTrackVolumeToCache(mixerId, isMasterOrPersonal); + updateLocalTrackMuteFromControlState(updated); + + if (context.trackVolumeObject && context.trackVolumeObject.broadcast) { + scheduleTrackSync(); + } + + pushRecord({ + kind: "webClientControlStateSet", + mixer_id: mixerId, + mode: modeKey(isMasterOrPersonal), + state: deepClone(updated), + broadcast: !!(context.trackVolumeObject && context.trackVolumeObject.broadcast) + }); + + return invokeDelegate("SessionSetControlState", arguments); + }; + + this.UpdateSessionInfo = function(sessionInfo) { + if (sessionInfo && sessionInfo.id) { + state.currentSessionId = sessionInfo.id; + } + + if (sessionInfo && sessionInfo.participants && sessionInfo.participants.length) { + var nextParticipants = {}; + sessionInfo.participants.forEach(function(participant) { + if (!participant || !participant.client_id) { return; } + nextParticipants[participant.client_id] = deepClone(participant); + }); + state.participants = nextParticipants; + } + + return invokeDelegate("UpdateSessionInfo", arguments); + }; + + this.ParticipantJoined = function(session, participant) { + if (participant && participant.client_id) { + state.participants[participant.client_id] = deepClone(participant); + if (state.webrtc.enabled) { + var selfClientId = getSelfClientId(); + if (participant.client_id !== selfClientId) { + maybeCreateOffer(participant.client_id); + } + } + } + return invokeDelegate("ParticipantJoined", arguments); + }; + + this.ParticipantLeft = function(session, participant) { + if (participant && participant.client_id) { + delete state.participants[participant.client_id]; + closePeerConnection(participant.client_id); + } + return invokeDelegate("ParticipantLeft", arguments); + }; + + this.ClientJoinedSession = function(sourceUserId, clientId, sessionId) { + if (clientId) { + state.participants[clientId] = state.participants[clientId] || {client_id: clientId}; + if (state.webrtc.enabled) { + var selfClientId = getSelfClientId(); + if (clientId !== selfClientId) { + maybeCreateOffer(clientId); + } + } + } + return invokeDelegate("ClientJoinedSession", arguments); + }; + + this.ClientLeftSession = function(sourceUserId, clientId, sessionId) { + if (clientId) { + delete state.participants[clientId]; + closePeerConnection(clientId); + } + return invokeDelegate("ClientLeftSession", arguments); + }; + + this.P2PMessageReceived = function(from, payload) { + var parsedPayload = payload; + if (typeof parsedPayload === "string") { + try { + parsedPayload = JSON.parse(parsedPayload); + } catch (e) {} + } + + if (parsedPayload && parsedPayload.jk_webclient_webrtc && parsedPayload.payload) { + onWebRtcSignal(from, parsedPayload.payload); + } + + return invokeDelegate("P2PMessageReceived", [from, payload]); + }; + // Keep a handle to the delegate for incremental replacement/testing. this.__delegate = inner; this.__state = state; this.__recorder = recorder; + state.webrtc.enabled = readWebRtcEnabledFlag(); + if (state.webrtc.enabled) { + pushRecord({kind: "webrtc.enabled"}); + } + recorder.enabled = recorderEnabledByDefault(); if (recorder.enabled) { pushRecord({kind: "webClientRecorder", action: "auto-enable"}); diff --git a/web/app/helpers/client_helper.rb b/web/app/helpers/client_helper.rb index 890d2049c..ea4d3da29 100644 --- a/web/app/helpers/client_helper.rb +++ b/web/app/helpers/client_helper.rb @@ -93,5 +93,7 @@ module ClientHelper gon.stripe_publishable_key = Rails.application.config.stripe[:publishable_key] gon.spa_origin_url = Rails.application.config.spa_origin_url gon.log_to_server = (ENV['LOG_TO_SERVER'].to_s == '1') + gon.legacy_qt_webkit_client = legacy_qt_webkit_client? + gon.show_mode_flags_overlay = Rails.env.development? end end diff --git a/web/package.json b/web/package.json index d2d76de53..4556c2fb0 100644 --- a/web/package.json +++ b/web/package.json @@ -1,16 +1,22 @@ { "name": "jam-web", - "dependencies" : { + "scripts": { + "test:playwright": "npx playwright test -c spec/playwright/playwright.config.js", + "test:playwright:headed": "HEADLESS=false npx playwright test -c spec/playwright/playwright.config.js" + }, + "dependencies": { "browserify": "~> 6.3", "browserify-incremental": "^1.4.0", + "coffee-reactify": "~>3.0.0", "coffeeify": "~> 0.6", - "reactify": "^0.17.1", - "coffee-reactify": "~>3.0.0", "react": "^0.12.0", - "react-tools": "^0.12.1" + "react-tools": "^0.12.1", + "reactify": "^0.17.1" }, - "engines": { "node": ">= 0.10" + }, + "devDependencies": { + "@playwright/test": "^1.58.2" } }