From c7220125e5eeacb1f4a5f14b263d7a234b9b25f4 Mon Sep 17 00:00:00 2001 From: Jan-Ivar Bruaroey Date: Fri, 15 Dec 2023 17:05:57 -0500 Subject: [PATCH 1/4] Update video-analyzer to spec. --- .../video-analyzer/js/main.js | 61 +++++-------------- .../video-analyzer/js/worker.js | 51 ++++++++++++++++ 2 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 src/content/insertable-streams/video-analyzer/js/worker.js diff --git a/src/content/insertable-streams/video-analyzer/js/main.js b/src/content/insertable-streams/video-analyzer/js/main.js index 77811d387d..6844d57a26 100644 --- a/src/content/insertable-streams/video-analyzer/js/main.js +++ b/src/content/insertable-streams/video-analyzer/js/main.js @@ -34,11 +34,8 @@ hdButton.addEventListener('click', () => { const banner = document.querySelector('#banner'); -const supportsInsertableStreams = - !!RTCRtpSender.prototype.createEncodedStreams; - -if (!supportsInsertableStreams) { - banner.innerText = 'Your browser does not support Insertable Streams. ' + +if (!window.RTCRtpScriptTransform) { + banner.innerText = 'Your browser does not support WebRTC encoded transforms. ' + 'This sample will not work.'; startButton.disabled = true; } @@ -62,6 +59,8 @@ remoteVideo.addEventListener('loadedmetadata', function() { let localStream; let pc1; let pc2; +const worker = new Worker("js/worker.js"); + const offerOptions = { offerToReceiveAudio: 0, offerToReceiveVideo: 1 @@ -170,11 +169,7 @@ function onSetSessionDescriptionError(error) { function gotRemoteTrack(e) { console.log('pc2 received remote stream'); - const frameStreams = e.receiver.createEncodedStreams(); - frameStreams.readable.pipeThrough(new TransformStream({ - transform: videoAnalyzer - })) - .pipeTo(frameStreams.writable); + e.receiver.transform = new RTCRtpScriptTransform(worker, {}); remoteVideo.srcObject = e.streams[0]; } @@ -237,49 +232,21 @@ const interFrameCountDisplay = document.querySelector('#interframe-count'); const interFrameSizeDisplay = document.querySelector('#interframe-size'); const videoSizeDisplay = document.querySelector('#video-size'); const duplicateCountDisplay = document.querySelector('#duplicate-count'); -let keyFrameCount = 0; -let interFrameCount = 0; -let keyFrameLastSize = 0; -let interFrameLastSize = 0; -let duplicateCount = 0; -let prevFrameType; -let prevFrameTimestamp; -let prevFrameSynchronizationSource; - -function videoAnalyzer(encodedFrame, controller) { - const view = new DataView(encodedFrame.data); - // We assume that the video is VP8. - // TODO: Check the codec to see that it is. - // The lowest value bit in the first byte is the keyframe indicator. - // https://tools.ietf.org/html/rfc6386#section-9.1 - const keyframeBit = view.getUint8(0) & 0x01; - // console.log(view.getUint8(0).toString(16)); - if (keyframeBit === 0) { - keyFrameCount++; - keyFrameLastSize = encodedFrame.data.byteLength; - } else { - interFrameCount++; - interFrameLastSize = encodedFrame.data.byteLength; - } - if (encodedFrame.type === prevFrameType && - encodedFrame.timestamp === prevFrameTimestamp && - encodedFrame.synchronizationSource === prevFrameSynchronizationSource) { - duplicateCount++; - } - prevFrameType = encodedFrame.type; - prevFrameTimestamp = encodedFrame.timestamp; - prevFrameSynchronizationSource = encodedFrame.synchronizationSource; - controller.enqueue(encodedFrame); -} -// Update the display of the counters once a second. -setInterval(() => { +worker.onmessage = ({data: { + keyFrameCount, + interFrameCount, + keyFrameLastSize, + interFrameLastSize, + duplicateCount +}}) => { + // Update the display of the counters keyFrameCountDisplay.innerText = keyFrameCount; keyFrameSizeDisplay.innerText = keyFrameLastSize; interFrameCountDisplay.innerText = interFrameCount; interFrameSizeDisplay.innerText = interFrameLastSize; duplicateCountDisplay.innerText = duplicateCount; -}, 500); +}; remoteVideo.addEventListener('resize', () => { console.log(`Remote video size changed to ${remoteVideo.videoWidth}x${remoteVideo.videoHeight}`); diff --git a/src/content/insertable-streams/video-analyzer/js/worker.js b/src/content/insertable-streams/video-analyzer/js/worker.js new file mode 100644 index 0000000000..c386d95158 --- /dev/null +++ b/src/content/insertable-streams/video-analyzer/js/worker.js @@ -0,0 +1,51 @@ +onrtctransform = async ({transformer: {readable, writable, options}}) => { + console.log(`Started ${options.side} transform`); + + await readable.pipeThrough(new TransformStream({ + transform: videoAnalyzer + })).pipeTo(writable); +} + +let keyFrameCount = 0; +let interFrameCount = 0; +let keyFrameLastSize = 0; +let interFrameLastSize = 0; +let duplicateCount = 0; +let prevFrameType; +let prevFrameTimestamp; +let prevFrameSynchronizationSource; + +function videoAnalyzer(encodedFrame, controller) { + const view = new DataView(encodedFrame.data); + // We assume that the video is VP8. + // TODO: Check the codec to see that it is. + // The lowest value bit in the first byte is the keyframe indicator. + // https://tools.ietf.org/html/rfc6386#section-9.1 + const keyframeBit = view.getUint8(0) & 0x01; + // console.log(view.getUint8(0).toString(16)); + if (keyframeBit === 0) { + keyFrameCount++; + keyFrameLastSize = encodedFrame.data.byteLength; + } else { + interFrameCount++; + interFrameLastSize = encodedFrame.data.byteLength; + } + if (encodedFrame.type === prevFrameType && + encodedFrame.timestamp === prevFrameTimestamp && + encodedFrame.synchronizationSource === prevFrameSynchronizationSource) { + duplicateCount++; + } + prevFrameType = encodedFrame.type; + prevFrameTimestamp = encodedFrame.timestamp; + prevFrameSynchronizationSource = encodedFrame.synchronizationSource; + controller.enqueue(encodedFrame); +} + +// Update the display of the counters once a second. +setInterval(() => self.postMessage({ + keyFrameCount, + keyFrameLastSize, + interFrameCount, + interFrameLastSize, + duplicateCount +}), 500); From 84c19f34ef71a19651d541f0ba49ae6c88df181e Mon Sep 17 00:00:00 2001 From: Jan-Ivar Bruaroey Date: Fri, 15 Dec 2023 17:24:16 -0500 Subject: [PATCH 2/4] Fix GitHub Actions lint errors --- src/content/insertable-streams/video-analyzer/js/main.js | 2 +- src/content/insertable-streams/video-analyzer/js/worker.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/content/insertable-streams/video-analyzer/js/main.js b/src/content/insertable-streams/video-analyzer/js/main.js index 6844d57a26..8a2ed7b5af 100644 --- a/src/content/insertable-streams/video-analyzer/js/main.js +++ b/src/content/insertable-streams/video-analyzer/js/main.js @@ -59,7 +59,7 @@ remoteVideo.addEventListener('loadedmetadata', function() { let localStream; let pc1; let pc2; -const worker = new Worker("js/worker.js"); +const worker = new Worker('js/worker.js'); const offerOptions = { offerToReceiveAudio: 0, diff --git a/src/content/insertable-streams/video-analyzer/js/worker.js b/src/content/insertable-streams/video-analyzer/js/worker.js index c386d95158..54c065756b 100644 --- a/src/content/insertable-streams/video-analyzer/js/worker.js +++ b/src/content/insertable-streams/video-analyzer/js/worker.js @@ -4,7 +4,7 @@ onrtctransform = async ({transformer: {readable, writable, options}}) => { await readable.pipeThrough(new TransformStream({ transform: videoAnalyzer })).pipeTo(writable); -} +}; let keyFrameCount = 0; let interFrameCount = 0; From b6fcbad84905e0009edf10214f7d983cbfa59604 Mon Sep 17 00:00:00 2001 From: Jan-Ivar Bruaroey Date: Fri, 15 Dec 2023 20:07:10 -0500 Subject: [PATCH 3/4] Remove a console.log statement --- src/content/insertable-streams/video-analyzer/js/worker.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/content/insertable-streams/video-analyzer/js/worker.js b/src/content/insertable-streams/video-analyzer/js/worker.js index 54c065756b..6eddbef1df 100644 --- a/src/content/insertable-streams/video-analyzer/js/worker.js +++ b/src/content/insertable-streams/video-analyzer/js/worker.js @@ -1,6 +1,4 @@ onrtctransform = async ({transformer: {readable, writable, options}}) => { - console.log(`Started ${options.side} transform`); - await readable.pipeThrough(new TransformStream({ transform: videoAnalyzer })).pipeTo(writable); From fee52c645434db6670eede2c1df7d5b90e75fecd Mon Sep 17 00:00:00 2001 From: Jan-Ivar Bruaroey Date: Fri, 15 Dec 2023 20:23:21 -0500 Subject: [PATCH 4/4] Accept shimming. --- src/content/insertable-streams/video-analyzer/js/worker.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/content/insertable-streams/video-analyzer/js/worker.js b/src/content/insertable-streams/video-analyzer/js/worker.js index 6eddbef1df..8ca1ed4fea 100644 --- a/src/content/insertable-streams/video-analyzer/js/worker.js +++ b/src/content/insertable-streams/video-analyzer/js/worker.js @@ -4,6 +4,9 @@ onrtctransform = async ({transformer: {readable, writable, options}}) => { })).pipeTo(writable); }; +// accept shimming +onmessage = ({data}) => data.rtctransform && onrtctransform({transformer: data.rtctransform}); + let keyFrameCount = 0; let interFrameCount = 0; let keyFrameLastSize = 0;