Question about Multiple Video Codec Support in SFU Scenario

[San9H0]

Description

We are implementing an SFU (Selective Forwarding Unit) that needs to handle multiple streamers using different video codecs.

Our Use Case

• SFU acts as an offerer
• Streamer1 connects using VP8
• Viewer connects and receives VP8 stream successfully
• Streamer2 tries to connect using H264
• SFU attempts to add new transceiver with H264 codec
• Renegotiation fails

Question

Is this limitation (not being able to add a different codec during renegotiation) intentional? If so, what would be the recommended approach for handling multiple codecs in an SFU scenario?

Current Workaround

We currently maintain a fork with modifications to handle this case, but we'd prefer to use the upstream version if possible. we've added an interface function to push codecs into MediaEngine's negotiatedVideoCodecs and negotiatedAudioCodecs to handle this scenario.

Scenario

func TestPeerConnection_Renegotiation_AddTransceiver_With_Different_Codec(t *testing.T) {
	lim := test.TimeOut(time.Second * 30)
	defer lim.Stop()

	report := test.CheckRoutines(t)
	defer report()

	// Our SFU server is on the offer side. Our SFU server supports VP8 and H264
	pcOffer, err := NewPeerConnection(Configuration{})
	assert.NoError(t, err)

	// Viewer is on the answer side
	pcAnswer, err := NewPeerConnection(Configuration{})
	assert.NoError(t, err)

	tracksCh := make(chan *TrackRemote)
	pcAnswer.OnTrack(func(track *TrackRemote, _ *RTPReceiver) {
		tracksCh <- track
		for {
			if _, _, readErr := track.ReadRTP(); errors.Is(readErr, io.EOF) {
				return
			}
		}
	})
	connected := make(chan struct{})
	pcOffer.OnConnectionStateChange(func(state PeerConnectionState) {
		if state == PeerConnectionStateConnected {
			close(connected)
		}
	})

	err = signalPair(pcOffer, pcAnswer)
	<-connected
	require.NoError(t, err)

	track1, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, "video1", "pion1")
	require.NoError(t, err)

	// First streamer uses VP8
	sender1, err := pcOffer.AddTransceiverFromTrack(track1)
	_ = sender1
	require.NoError(t, err)
	err = sender1.SetCodecPreferences([]RTPCodecParameters{
		{
			RTPCodecCapability: RTPCodecCapability{MimeType: MimeTypeVP8, ClockRate: 90000},
			PayloadType:        96,
		},
	})
	require.NoError(t, err)

	err = signalPair(pcOffer, pcAnswer)

	transceivers := pcOffer.GetTransceivers()
	require.Equal(t, 1, len(transceivers))
	require.Equal(t, "1", transceivers[0].Mid())

	transceivers = pcAnswer.GetTransceivers()
	require.Equal(t, 1, len(transceivers))
	require.Equal(t, "1", transceivers[0].Mid())

	ctx, cancel := context.WithCancel(context.Background())
	go sendVideoUntilDone(t, ctx.Done(), []*TrackLocalStaticSample{track1})
	remoteTrack1 := <-tracksCh
	cancel()

	assert.Equal(t, "video1", remoteTrack1.ID())
	assert.Equal(t, "pion1", remoteTrack1.StreamID())

	// Second streamer is created
	track2, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeH264}, "video2", "pion2")
	// track2, err := NewTrackLocalStaticSample(RTPCodecCapability{MimeType: MimeTypeVP8}, "video2", "pion2")
	require.NoError(t, err)

	// Second streamer uses H264
	sender2, err := pcOffer.AddTransceiverFromTrack(track2)
	_ = sender2
	require.NoError(t, err)
	err = sender2.SetCodecPreferences([]RTPCodecParameters{
		{
			RTPCodecCapability: RTPCodecCapability{MimeType: MimeTypeH264, ClockRate: 90000, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f"},
			// RTPCodecCapability: RTPCodecCapability{MimeType: MimeTypeVP8, ClockRate: 90000, SDPFmtpLine: ""},
			PayloadType: 112,
		},
	})
	require.NoError(t, err) // Error occurs here

	require.NoError(t, signalPair(pcOffer, pcAnswer))
	ctx, cancel = context.WithCancel(context.Background())
	go sendVideoUntilDone(t, ctx.Done(), []*TrackLocalStaticSample{track2})

	remoteTrack2 := <-tracksCh
	cancel()

	assert.Equal(t, "video2", remoteTrack2.ID())
	assert.Equal(t, "pion2", remoteTrack2.StreamID())

	closePairNow(t, pcOffer, pcAnswer)
}

[JoeTurki]

Hello there is a working PR to add multiple codec negotiation #3018 Expect it to be merged soon 👍

[San9H0]

Thank you for the quick response and pointing to PR #3018.

However, looking at the PR, it seems to focus on the answer side (RemoteDescription). In our SFU scenario, we are on the offer side and need to support multiple codecs in the offer SDP during CreateOffer.

Our SFU acts as an offerer and needs to generate an SDP that includes both VP8 and H264 codecs when creating the offer. The current PR doesn't seem to address this specific use case.