Add TCP+Relay detection event (#485)

* Change the label for the remote candidate
* Added the following missing WebRTC stats id,timestamp,type,lastPacketReceivedTimestamp,lastPacketSentTimestamp,priority,remoteCandidateId,transportId and writable
* Added relayProtocol and transport ID to the candidate Stat
* Refactored the candidate pair to be an array as many pairs are generated
* Implemeneted a WebRtcTCPRelayDetectedEvent that is fired when WebRTC transport is TCP AND we are using a relay candidate (e.g. a TURN server is being used).
* Refactored the stats panel to use the `getActiveCandidatePair()`

Author: david-macpherson

Frontend/library/src/PeerConnectionController/AggregatedStats.ts

@@ -25,7 +25,7 @@ export class AggregatedStats {
     inboundAudioStats: InboundAudioStats;
     lastVideoStats: InboundVideoStats;
     lastAudioStats: InboundAudioStats;
-    candidatePair: CandidatePairStats;
+    candidatePairs: Array<CandidatePairStats>;
     DataChannelStats: DataChannelStats;
     localCandidates: Array<CandidateStat>;
     remoteCandidates: Array<CandidateStat>;
@@ -37,7 +37,6 @@ export class AggregatedStats {
     constructor() {
         this.inboundVideoStats = new InboundVideoStats();
         this.inboundAudioStats = new InboundAudioStats();
-        this.candidatePair = new CandidatePairStats();
         this.DataChannelStats = new DataChannelStats();
         this.outBoundVideoStats = new OutBoundVideoStats();
         this.sessionStats = new SessionStats();
@@ -52,6 +51,7 @@ export class AggregatedStats {
     processStats(rtcStatsReport: RTCStatsReport) {
         this.localCandidates = new Array<CandidateStat>();
         this.remoteCandidates = new Array<CandidateStat>();
+        this.candidatePairs = new Array<CandidatePairStats>();
 
         rtcStatsReport.forEach((stat) => {
             const type: RTCStatsTypePS = stat.type;
@@ -120,16 +120,10 @@ export class AggregatedStats {
      * @param stat - the stats coming in from ice candidates
      */
     handleCandidatePair(stat: CandidatePairStats) {
-        this.candidatePair.bytesReceived = stat.bytesReceived;
-        this.candidatePair.bytesSent = stat.bytesSent;
-        this.candidatePair.localCandidateId = stat.localCandidateId;
-        this.candidatePair.remoteCandidateId = stat.remoteCandidateId;
-        this.candidatePair.nominated = stat.nominated;
-        this.candidatePair.readable = stat.readable;
-        this.candidatePair.selected = stat.selected;
-        this.candidatePair.writable = stat.writable;
-        this.candidatePair.state = stat.state;
-        this.candidatePair.currentRoundTripTime = stat.currentRoundTripTime;
+
+        // Add the candidate pair to the candidate pair array
+        this.candidatePairs.push(stat)
+
     }
 
     /**
@@ -162,6 +156,8 @@ export class AggregatedStats {
         localCandidate.protocol = stat.protocol;
         localCandidate.candidateType = stat.candidateType;
         localCandidate.id = stat.id;
+        localCandidate.relayProtocol = stat.relayProtocol;
+        localCandidate.transportId = stat.transportId;
         this.localCandidates.push(localCandidate);
     }
 
@@ -171,12 +167,14 @@ export class AggregatedStats {
      */
     handleRemoteCandidate(stat: CandidateStat) {
         const RemoteCandidate = new CandidateStat();
-        RemoteCandidate.label = 'local-candidate';
+        RemoteCandidate.label = 'remote-candidate';
         RemoteCandidate.address = stat.address;
         RemoteCandidate.port = stat.port;
         RemoteCandidate.protocol = stat.protocol;
         RemoteCandidate.id = stat.id;
         RemoteCandidate.candidateType = stat.candidateType;
+        RemoteCandidate.relayProtocol = stat.relayProtocol;
+        RemoteCandidate.transportId = stat.transportId
         this.remoteCandidates.push(RemoteCandidate);
     }
 
@@ -308,4 +306,12 @@ export class AggregatedStats {
     isNumber(value: unknown): boolean {
         return typeof value === 'number' && isFinite(value);
     }
+
+    /**
+     * Helper function to return the active candidate pair
+     * @returns The candidate pair that is currently receiving data
+     */
+    public getActiveCandidatePair(): CandidatePairStats | null {
+        return this.candidatePairs.find((candidatePair) => candidatePair.bytesReceived > 0, null)
+    }  
 }

Frontend/library/src/PeerConnectionController/CandidatePairStats.ts

@@ -6,12 +6,19 @@
 export class CandidatePairStats {
     bytesReceived: number;
     bytesSent: number;
+    currentRoundTripTime: number;
+    id: string;
+    lastPacketReceivedTimestamp: number;
+    lastPacketSentTimestamp: number;
     localCandidateId: string;
-    remoteCandidateId: string;
     nominated: boolean;
+    priority: number;
     readable: boolean;
-    writable: boolean;
+    remoteCandidateId: string;
     selected: boolean;
     state: string;
-    currentRoundTripTime: number;
+    timestamp: number;
+    transportId: string;
+    type: string;
+    writable: boolean;  
 }

Frontend/library/src/PeerConnectionController/CandidateStat.ts

@@ -4,10 +4,12 @@
  * ICE Candidate Stat collected from the RTC Stats Report
  */
 export class CandidateStat {
-    label: string;
-    id: string;
     address: string;
     candidateType: string;
+    id: string;
+    label: string;    
     port: number;
     protocol: 'tcp' | 'udp';
+    relayProtocol: 'tcp' | 'udp' | 'tls';
+    transportId: string;
 }

Frontend/library/src/PixelStreaming/PixelStreaming.test.ts

@@ -398,9 +398,9 @@ describe('PixelStreaming', () => {
             expect.objectContaining({
                 data: {
                     aggregatedStats: expect.objectContaining({
-                        candidatePair: expect.objectContaining({
-                            bytesReceived: 123
-                        }),
+                        candidatePairs: [
+                            expect.objectContaining({ bytesReceived: 123 })
+                        ],
                         localCandidates: [
                             expect.objectContaining({ address: 'mock-address' })
                         ]

Frontend/library/src/PixelStreaming/PixelStreaming.ts

@@ -28,7 +28,8 @@ import {
     WebRtcSdpEvent,
     DataChannelLatencyTestResponseEvent,
     DataChannelLatencyTestResultEvent,
-    PlayerCountEvent
+    PlayerCountEvent,
+    WebRtcTCPRelayDetectedEvent
 } from '../Util/EventEmitter';
 import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive';
 import { WebXRController } from '../WebXR/WebXRController';
@@ -62,6 +63,7 @@ export class PixelStreaming {
     protected _webRtcController: WebRtcPlayerController;
     protected _webXrController: WebXRController;
     protected _dataChannelLatencyTestController: DataChannelLatencyTestController;
+
     /**
      * Configuration object. You can read or modify config through this object. Whenever
      * the configuration is changed, the library will emit a `settingsChanged` event.
@@ -116,6 +118,13 @@ export class PixelStreaming {
             this.onScreenKeyboardHelper.showOnScreenKeyboard(command);
 
         this._webXrController = new WebXRController(this._webRtcController);
+
+        // Add event listener for the webRtcConnected event
+        this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => {
+
+            // Bind to the stats received event
+            this._eventEmitter.addEventListener("statsReceived",  this._setupWebRtcTCPRelayDetection.bind(this));
+        });
     }
 
     /**
@@ -627,6 +636,28 @@ export class PixelStreaming {
         );
     }
 
+    // Sets up to emit the webrtc tcp relay detect event 
+    _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent) {
+        // Get the active candidate pair
+        let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair();
+                
+        // Check if the active candidate pair is not null
+        if (activeCandidatePair != null) {
+
+            // Get the local candidate assigned to the active candidate pair
+            let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null)
+
+            // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp
+            if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') {
+
+                // Send the web rtc tcp relay detected event
+                this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent());
+            }
+            // The check is completed and the stats listen event can be removed
+            this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection);
+        }
+    }
+
     /**
      * Request a connection latency test.
      * NOTE: There are plans to refactor all request* functions. Expect changes if you use this!

Frontend/library/src/Util/EventEmitter.ts

@@ -537,6 +537,16 @@ export class PlayerCountEvent extends Event {
     }
 }
 
+/**
+ * An event that is emitted when the webRTC connections is relayed over TCP.
+ */
+export class WebRtcTCPRelayDetectedEvent extends Event {
+    readonly type: 'webRtcTCPRelayDetected';
+    constructor() {
+        super('webRtcTCPRelayDetected');
+    }
+}
+
 export type PixelStreamingEvent =
     | AfkWarningActivateEvent
     | AfkWarningUpdateEvent
@@ -573,7 +583,8 @@ export type PixelStreamingEvent =
     | XrSessionStartedEvent
     | XrSessionEndedEvent
     | XrFrameEvent
-    | PlayerCountEvent;
+    | PlayerCountEvent
+    | WebRtcTCPRelayDetectedEvent;
 
 export class EventEmitter extends EventTarget {
     /**

Frontend/ui-library/src/Application/Application.ts

@@ -378,6 +378,14 @@ export class Application {
             ({ data: { count }}) => 
                 this.onPlayerCount(count)
         );
+        this.stream.addEventListener(
+            'webRtcTCPRelayDetected', 
+            ({}) => 
+                Logger.Warning(
+                    Logger.GetStackTrace(),
+                    `Stream quailty degraded due to network enviroment, stream is relayed over TCP.`
+               )
+        );
     }
 
     /**

Frontend/ui-library/src/UI/StatsPanel.ts

@@ -1,7 +1,7 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 import { LatencyTest } from './LatencyTest';
-import { InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4';
+import { CandidatePairStats, InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4';
 import { AggregatedStats } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4';
 import { MathUtils } from '../Util/MathUtils';
 import {DataChannelLatencyTest} from "./DataChannelLatencyTest";
@@ -318,14 +318,17 @@ export class StatsPanel {
             );
         }
 
+        // Store the active candidate pair return a new Candidate pair stat if getActiveCandidate is null
+        let activeCandidatePair = stats.getActiveCandidatePair() != null ? stats.getActiveCandidatePair() : new CandidatePairStats();
+
         // RTT
         const netRTT =
             Object.prototype.hasOwnProperty.call(
-                stats.candidatePair,
+                activeCandidatePair,
                 'currentRoundTripTime'
-            ) && stats.isNumber(stats.candidatePair.currentRoundTripTime)
+            ) && stats.isNumber(activeCandidatePair.currentRoundTripTime)
                 ? numberFormat.format(
-                      stats.candidatePair.currentRoundTripTime * 1000
+                    activeCandidatePair.currentRoundTripTime * 1000
                   )
                 : "Can't calculate";
         this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT);