/* * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "call/rtp_video_sender.h" #include #include #include #include "call/rtp_transport_controller_send.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtcp_packet/nack.h" #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/video_coding/fec_controller_default.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/rate_limiter.h" #include "test/field_trial.h" #include "test/gmock.h" #include "test/gtest.h" #include "test/mock_frame_transformer.h" #include "test/mock_transport.h" #include "test/scenario/scenario.h" #include "test/time_controller/simulated_time_controller.h" #include "video/call_stats.h" #include "video/send_delay_stats.h" #include "video/send_statistics_proxy.h" using ::testing::_; using ::testing::NiceMock; using ::testing::SaveArg; using ::testing::SizeIs; namespace webrtc { namespace { const int8_t kPayloadType = 96; const uint32_t kSsrc1 = 12345; const uint32_t kSsrc2 = 23456; const uint32_t kRtxSsrc1 = 34567; const uint32_t kRtxSsrc2 = 45678; const int16_t kInitialPictureId1 = 222; const int16_t kInitialPictureId2 = 44; const int16_t kInitialTl0PicIdx1 = 99; const int16_t kInitialTl0PicIdx2 = 199; const int64_t kRetransmitWindowSizeMs = 500; const int kTransportsSequenceExtensionId = 7; const int kDependencyDescriptorExtensionId = 8; class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { public: MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override)); }; RtpSenderObservers CreateObservers( RtcpRttStats* rtcp_rtt_stats, RtcpIntraFrameObserver* intra_frame_callback, RtcpStatisticsCallback* rtcp_stats, ReportBlockDataObserver* report_block_data_observer, StreamDataCountersCallback* rtp_stats, BitrateStatisticsObserver* bitrate_observer, FrameCountObserver* frame_count_observer, RtcpPacketTypeCounterObserver* rtcp_type_observer, SendSideDelayObserver* send_delay_observer, SendPacketObserver* send_packet_observer) { RtpSenderObservers observers; observers.rtcp_rtt_stats = rtcp_rtt_stats; observers.intra_frame_callback = intra_frame_callback; observers.rtcp_loss_notification_observer = nullptr; observers.rtcp_stats = rtcp_stats; observers.report_block_data_observer = report_block_data_observer; observers.rtp_stats = rtp_stats; observers.bitrate_observer = bitrate_observer; observers.frame_count_observer = frame_count_observer; observers.rtcp_type_observer = rtcp_type_observer; observers.send_delay_observer = send_delay_observer; observers.send_packet_observer = send_packet_observer; return observers; } BitrateConstraints GetBitrateConfig() { BitrateConstraints bitrate_config; bitrate_config.min_bitrate_bps = 30000; bitrate_config.start_bitrate_bps = 300000; bitrate_config.max_bitrate_bps = 3000000; return bitrate_config; } VideoSendStream::Config CreateVideoSendStreamConfig( Transport* transport, const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type) { VideoSendStream::Config config(transport); config.rtp.ssrcs = ssrcs; config.rtp.rtx.ssrcs = rtx_ssrcs; config.rtp.payload_type = payload_type; config.rtp.rtx.payload_type = payload_type + 1; config.rtp.nack.rtp_history_ms = 1000; config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri, kTransportsSequenceExtensionId); config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::kUri, kDependencyDescriptorExtensionId); return config; } class RtpVideoSenderTestFixture { public: RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states, FrameCountObserver* frame_count_observer, rtc::scoped_refptr frame_transformer) : time_controller_(Timestamp::Millis(1000000)), config_(CreateVideoSendStreamConfig(&transport_, ssrcs, rtx_ssrcs, payload_type)), send_delay_stats_(time_controller_.GetClock()), bitrate_config_(GetBitrateConfig()), transport_controller_( time_controller_.GetClock(), &event_log_, nullptr, nullptr, bitrate_config_, time_controller_.CreateProcessThread("PacerThread"), time_controller_.GetTaskQueueFactory(), &field_trials_), process_thread_(time_controller_.CreateProcessThread("test_thread")), call_stats_(time_controller_.GetClock(), process_thread_.get()), stats_proxy_(time_controller_.GetClock(), config_, VideoEncoderConfig::ContentType::kRealtimeVideo), retransmission_rate_limiter_(time_controller_.GetClock(), kRetransmitWindowSizeMs) { std::map suspended_ssrcs; router_ = std::make_unique( time_controller_.GetClock(), suspended_ssrcs, suspended_payload_states, config_.rtp, config_.rtcp_report_interval_ms, &transport_, CreateObservers(&call_stats_, &encoder_feedback_, &stats_proxy_, &stats_proxy_, &stats_proxy_, &stats_proxy_, frame_count_observer, &stats_proxy_, &stats_proxy_, &send_delay_stats_), &transport_controller_, &event_log_, &retransmission_rate_limiter_, std::make_unique(time_controller_.GetClock()), nullptr, CryptoOptions{}, frame_transformer); } RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states, FrameCountObserver* frame_count_observer) : RtpVideoSenderTestFixture(ssrcs, rtx_ssrcs, payload_type, suspended_payload_states, frame_count_observer, /*frame_transformer=*/nullptr) {} RtpVideoSenderTestFixture( const std::vector& ssrcs, const std::vector& rtx_ssrcs, int payload_type, const std::map& suspended_payload_states) : RtpVideoSenderTestFixture(ssrcs, rtx_ssrcs, payload_type, suspended_payload_states, /*frame_count_observer=*/nullptr, /*frame_transformer=*/nullptr) {} RtpVideoSender* router() { return router_.get(); } MockTransport& transport() { return transport_; } void AdvanceTime(TimeDelta delta) { time_controller_.AdvanceTime(delta); } private: NiceMock transport_; NiceMock encoder_feedback_; GlobalSimulatedTimeController time_controller_; RtcEventLogNull event_log_; VideoSendStream::Config config_; SendDelayStats send_delay_stats_; BitrateConstraints bitrate_config_; const FieldTrialBasedConfig field_trials_; RtpTransportControllerSend transport_controller_; std::unique_ptr process_thread_; // TODO(tommi): Use internal::CallStats. CallStats call_stats_; SendStatisticsProxy stats_proxy_; RateLimiter retransmission_rate_limiter_; std::unique_ptr router_; }; } // namespace TEST(RtpVideoSenderTest, SendOnOneModule) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); EXPECT_NE( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); test.router()->SetActive(true); EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); test.router()->SetActive(false); EXPECT_NE( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); test.router()->SetActive(true); EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); } TEST(RtpVideoSenderTest, SendSimulcastSetActive) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image_1; encoded_image_1.SetTimestamp(1); encoded_image_1.capture_time_ms_ = 2; encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; test.router()->SetActive(true); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); EncodedImage encoded_image_2(encoded_image_1); encoded_image_2.SetSpatialIndex(1); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_2, &codec_info, nullptr) .error); // Inactive. test.router()->SetActive(false); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_2, &codec_info, nullptr) .error); } // Tests how setting individual rtp modules to active affects the overall // behavior of the payload router. First sets one module to active and checks // that outgoing data can be sent on this module, and checks that no data can // be sent if both modules are inactive. TEST(RtpVideoSenderTest, SendSimulcastSetActiveModules) { constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image_1; encoded_image_1.SetTimestamp(1); encoded_image_1.capture_time_ms_ = 2; encoded_image_1._frameType = VideoFrameType::kVideoFrameKey; encoded_image_1.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); EncodedImage encoded_image_2(encoded_image_1); encoded_image_2.SetSpatialIndex(1); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); CodecSpecificInfo codec_info; codec_info.codecType = kVideoCodecVP8; // Only setting one stream to active will still set the payload router to // active and allow sending data on the active stream. std::vector active_modules({true, false}); test.router()->SetActiveModules(active_modules); EXPECT_EQ(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); // Setting both streams to inactive will turn the payload router to // inactive. active_modules = {false, false}; test.router()->SetActiveModules(active_modules); // An incoming encoded image will not ask the module to send outgoing data // because the payload router is inactive. EXPECT_NE(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); EXPECT_NE(EncodedImageCallback::Result::OK, test.router() ->OnEncodedImage(encoded_image_1, &codec_info, nullptr) .error); } TEST(RtpVideoSenderTest, CreateWithNoPreviousStates) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.router()->SetActive(true); std::map initial_states = test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_NE(initial_states.find(kSsrc1), initial_states.end()); EXPECT_NE(initial_states.find(kSsrc2), initial_states.end()); } TEST(RtpVideoSenderTest, CreateWithPreviousStates) { const int64_t kState1SharedFrameId = 123; const int64_t kState2SharedFrameId = 234; RtpPayloadState state1; state1.picture_id = kInitialPictureId1; state1.tl0_pic_idx = kInitialTl0PicIdx1; state1.shared_frame_id = kState1SharedFrameId; RtpPayloadState state2; state2.picture_id = kInitialPictureId2; state2.tl0_pic_idx = kInitialTl0PicIdx2; state2.shared_frame_id = kState2SharedFrameId; std::map states = {{kSsrc1, state1}, {kSsrc2, state2}}; RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, states); test.router()->SetActive(true); std::map initial_states = test.router()->GetRtpPayloadStates(); EXPECT_EQ(2u, initial_states.size()); EXPECT_EQ(kInitialPictureId1, initial_states[kSsrc1].picture_id); EXPECT_EQ(kInitialTl0PicIdx1, initial_states[kSsrc1].tl0_pic_idx); EXPECT_EQ(kInitialPictureId2, initial_states[kSsrc2].picture_id); EXPECT_EQ(kInitialTl0PicIdx2, initial_states[kSsrc2].tl0_pic_idx); EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc1].shared_frame_id); EXPECT_EQ(kState2SharedFrameId, initial_states[kSsrc2].shared_frame_id); } TEST(RtpVideoSenderTest, FrameCountCallbacks) { class MockFrameCountObserver : public FrameCountObserver { public: MOCK_METHOD(void, FrameCountUpdated, (const FrameCounts& frame_counts, uint32_t ssrc), (override)); } callback; RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}, &callback); constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); encoded_image._frameType = VideoFrameType::kVideoFrameKey; // No callbacks when not active. EXPECT_CALL(callback, FrameCountUpdated).Times(0); EXPECT_NE( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); ::testing::Mock::VerifyAndClearExpectations(&callback); test.router()->SetActive(true); FrameCounts frame_counts; EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) .WillOnce(SaveArg<0>(&frame_counts)); EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); EXPECT_EQ(1, frame_counts.key_frames); EXPECT_EQ(0, frame_counts.delta_frames); ::testing::Mock::VerifyAndClearExpectations(&callback); encoded_image._frameType = VideoFrameType::kVideoFrameDelta; EXPECT_CALL(callback, FrameCountUpdated(_, kSsrc1)) .WillOnce(SaveArg<0>(&frame_counts)); EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); EXPECT_EQ(1, frame_counts.key_frames); EXPECT_EQ(1, frame_counts.delta_frames); } // Integration test verifying that ack of packet via TransportFeedback means // that the packet is removed from RtpPacketHistory and won't be retransmitted // again. TEST(RtpVideoSenderTest, DoesNotRetrasmitAckedPackets) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.router()->SetActive(true); constexpr uint8_t kPayload = 'a'; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData(EncodedImageBuffer::Create(&kPayload, 1)); // Send two tiny images, mapping to two RTP packets. Capture sequence numbers. std::vector rtp_sequence_numbers; std::vector transport_sequence_numbers; EXPECT_CALL(test.transport(), SendRtp) .Times(2) .WillRepeatedly([&rtp_sequence_numbers, &transport_sequence_numbers]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); rtp_sequence_numbers.push_back(rtp_packet.SequenceNumber()); transport_sequence_numbers.push_back(options.packet_id); return true; }); EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); encoded_image.SetTimestamp(2); encoded_image.capture_time_ms_ = 3; EXPECT_EQ( EncodedImageCallback::Result::OK, test.router()->OnEncodedImage(encoded_image, nullptr, nullptr).error); test.AdvanceTime(TimeDelta::Millis(33)); // Construct a NACK message for requesting retransmission of both packet. rtcp::Nack nack; nack.SetMediaSsrc(kSsrc1); nack.SetPacketIds(rtp_sequence_numbers); rtc::Buffer nack_buffer = nack.Build(); std::vector retransmitted_rtp_sequence_numbers; EXPECT_CALL(test.transport(), SendRtp) .Times(2) .WillRepeatedly([&retransmitted_rtp_sequence_numbers]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Capture the retransmitted sequence number from the RTX header. rtc::ArrayView payload = rtp_packet.payload(); retransmitted_rtp_sequence_numbers.push_back( ByteReader::ReadBigEndian(payload.data())); return true; }); test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size()); test.AdvanceTime(TimeDelta::Millis(33)); // Verify that both packets were retransmitted. EXPECT_EQ(retransmitted_rtp_sequence_numbers, rtp_sequence_numbers); // Simulate transport feedback indicating fist packet received, next packet // lost (not other way around as that would trigger early retransmit). StreamFeedbackObserver::StreamPacketInfo lost_packet_feedback; lost_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[0]; lost_packet_feedback.ssrc = kSsrc1; lost_packet_feedback.received = false; StreamFeedbackObserver::StreamPacketInfo received_packet_feedback; received_packet_feedback.rtp_sequence_number = rtp_sequence_numbers[1]; received_packet_feedback.ssrc = kSsrc1; received_packet_feedback.received = true; test.router()->OnPacketFeedbackVector( {lost_packet_feedback, received_packet_feedback}); // Advance time to make sure retransmission would be allowed and try again. // This time the retransmission should not happen for the first packet since // the history has been notified of the ack and removed the packet. The // second packet, included in the feedback but not marked as received, should // still be retransmitted. test.AdvanceTime(TimeDelta::Millis(33)); EXPECT_CALL(test.transport(), SendRtp) .WillOnce([&lost_packet_feedback](const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Capture the retransmitted sequence number from the RTX header. rtc::ArrayView payload = rtp_packet.payload(); EXPECT_EQ(lost_packet_feedback.rtp_sequence_number, ByteReader::ReadBigEndian(payload.data())); return true; }); test.router()->DeliverRtcp(nack_buffer.data(), nack_buffer.size()); test.AdvanceTime(TimeDelta::Millis(33)); } // This tests that we utilize transport wide feedback to retransmit lost // packets. This is tested by dropping all ordirary packets from a "lossy" // stream send along with an secondary untouched stream. The transport wide // feedback packets from the secondary stream allows the sending side to // detect and retreansmit the lost packets from the lossy stream. TEST(RtpVideoSenderTest, RetransmitsOnTransportWideLossInfo) { int rtx_packets; test::Scenario s(test_info_); test::CallClientConfig call_conf; // Keeping the bitrate fixed to avoid RTX due to probing. call_conf.transport.rates.max_rate = DataRate::KilobitsPerSec(300); call_conf.transport.rates.start_rate = DataRate::KilobitsPerSec(300); test::NetworkSimulationConfig net_conf; net_conf.bandwidth = DataRate::KilobitsPerSec(300); auto send_node = s.CreateSimulationNode(net_conf); auto* callee = s.CreateClient("return", call_conf); auto* route = s.CreateRoutes(s.CreateClient("send", call_conf), {send_node}, callee, {s.CreateSimulationNode(net_conf)}); test::VideoStreamConfig lossy_config; lossy_config.source.framerate = 5; auto* lossy = s.CreateVideoStream(route->forward(), lossy_config); // The secondary stream acts a driver for transport feedback messages, // ensuring that lost packets on the lossy stream are retransmitted. s.CreateVideoStream(route->forward(), test::VideoStreamConfig()); send_node->router()->SetFilter([&](const EmulatedIpPacket& packet) { RtpPacket rtp; if (rtp.Parse(packet.data)) { // Drops all regular packets for the lossy stream and counts all RTX // packets. Since no packets are let trough, NACKs can't be triggered // by the receiving side. if (lossy->send()->UsingSsrc(rtp.Ssrc())) { return false; } else if (lossy->send()->UsingRtxSsrc(rtp.Ssrc())) { ++rtx_packets; } } return true; }); // Run for a short duration and reset counters to avoid counting RTX packets // from initial probing. s.RunFor(TimeDelta::Seconds(1)); rtx_packets = 0; int decoded_baseline = 0; callee->SendTask([&decoded_baseline, &lossy]() { decoded_baseline = lossy->receive()->GetStats().frames_decoded; }); s.RunFor(TimeDelta::Seconds(1)); // We expect both that RTX packets were sent and that an appropriate number of // frames were received. This is somewhat redundant but reduces the risk of // false positives in future regressions (e.g. RTX is send due to probing). EXPECT_GE(rtx_packets, 1); int frames_decoded = 0; callee->SendTask([&decoded_baseline, &frames_decoded, &lossy]() { frames_decoded = lossy->receive()->GetStats().frames_decoded - decoded_baseline; }); EXPECT_EQ(frames_decoded, 5); } // Integration test verifying that retransmissions are sent for packets which // can be detected as lost early, using transport wide feedback. TEST(RtpVideoSenderTest, EarlyRetransmits) { RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}); test.router()->SetActive(true); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image._frameType = VideoFrameType::kVideoFrameKey; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); encoded_image.SetSpatialIndex(0); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; // Send two tiny images, mapping to single RTP packets. Capture sequence // numbers. uint16_t frame1_rtp_sequence_number = 0; uint16_t frame1_transport_sequence_number = 0; EXPECT_CALL(test.transport(), SendRtp) .WillOnce( [&frame1_rtp_sequence_number, &frame1_transport_sequence_number]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); frame1_rtp_sequence_number = rtp_packet.SequenceNumber(); frame1_transport_sequence_number = options.packet_id; EXPECT_EQ(rtp_packet.Ssrc(), kSsrc1); return true; }); EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); uint16_t frame2_rtp_sequence_number = 0; uint16_t frame2_transport_sequence_number = 0; encoded_image.SetSpatialIndex(1); EXPECT_CALL(test.transport(), SendRtp) .WillOnce( [&frame2_rtp_sequence_number, &frame2_transport_sequence_number]( const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); frame2_rtp_sequence_number = rtp_packet.SequenceNumber(); frame2_transport_sequence_number = options.packet_id; EXPECT_EQ(rtp_packet.Ssrc(), kSsrc2); return true; }); EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); EXPECT_NE(frame1_transport_sequence_number, frame2_transport_sequence_number); // Inject a transport feedback where the packet for the first frame is lost, // expect a retransmission for it. EXPECT_CALL(test.transport(), SendRtp) .WillOnce([&frame1_rtp_sequence_number](const uint8_t* packet, size_t length, const PacketOptions& options) { RtpPacket rtp_packet; EXPECT_TRUE(rtp_packet.Parse(packet, length)); EXPECT_EQ(rtp_packet.Ssrc(), kRtxSsrc1); // Retransmitted sequence number from the RTX header should match // the lost packet. rtc::ArrayView payload = rtp_packet.payload(); EXPECT_EQ(ByteReader::ReadBigEndian(payload.data()), frame1_rtp_sequence_number); return true; }); StreamFeedbackObserver::StreamPacketInfo first_packet_feedback; first_packet_feedback.rtp_sequence_number = frame1_rtp_sequence_number; first_packet_feedback.ssrc = kSsrc1; first_packet_feedback.received = false; StreamFeedbackObserver::StreamPacketInfo second_packet_feedback; second_packet_feedback.rtp_sequence_number = frame2_rtp_sequence_number; second_packet_feedback.ssrc = kSsrc2; second_packet_feedback.received = true; test.router()->OnPacketFeedbackVector( {first_packet_feedback, second_packet_feedback}); // Wait for pacer to run and send the RTX packet. test.AdvanceTime(TimeDelta::Millis(33)); } TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.router()->SetActive(true); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { FrameDependencyTemplate().T(0).Dtis("S"), FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. // Send in key frame. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(0).Dtis("S").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(1)); EXPECT_TRUE( sent_packets.back().HasExtension()); // Send in delta frame. encoded_image._frameType = VideoFrameType::kVideoFrameDelta; codec_specific.template_structure = absl::nullopt; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(1).Dtis("D").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}}; EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_TRUE( sent_packets.back().HasExtension()); } TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.router()->SetActive(true); RtpHeaderExtensionMap extensions; extensions.Register( kDependencyDescriptorExtensionId); std::vector sent_packets; ON_CALL(test.transport(), SendRtp) .WillByDefault([&](const uint8_t* packet, size_t length, const PacketOptions& options) { sent_packets.emplace_back(&extensions); EXPECT_TRUE(sent_packets.back().Parse(packet, length)); return true; }); const uint8_t kPayload[1] = {'a'}; EncodedImage encoded_image; encoded_image.SetTimestamp(1); encoded_image.capture_time_ms_ = 2; encoded_image.SetEncodedData( EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); CodecSpecificInfo codec_specific; codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { FrameDependencyTemplate().T(0).Dtis("S"), FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. // Send in a key frame. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.generic_frame_info = GenericFrameInfo::Builder().T(0).Dtis("S").Build(); codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(1)); EXPECT_TRUE( sent_packets.back().HasExtension()); // Send in a new key frame without the support for the dependency descriptor. encoded_image._frameType = VideoFrameType::kVideoFrameKey; codec_specific.template_structure = absl::nullopt; EXPECT_EQ(test.router() ->OnEncodedImage(encoded_image, &codec_specific, nullptr) .error, EncodedImageCallback::Result::OK); test.AdvanceTime(TimeDelta::Millis(33)); ASSERT_THAT(sent_packets, SizeIs(2)); EXPECT_FALSE( sent_packets.back().HasExtension()); } TEST(RtpVideoSenderTest, CanSetZeroBitrateWithOverhead) { test::ScopedFieldTrials trials("WebRTC-SendSideBwe-WithOverhead/Enabled/"); RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); BitrateAllocationUpdate update; update.target_bitrate = DataRate::Zero(); update.packet_loss_ratio = 0; update.round_trip_time = TimeDelta::Zero(); test.router()->OnBitrateUpdated(update, /*framerate*/ 0); } TEST(RtpVideoSenderTest, CanSetZeroBitrateWithoutOverhead) { RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}); BitrateAllocationUpdate update; update.target_bitrate = DataRate::Zero(); update.packet_loss_ratio = 0; update.round_trip_time = TimeDelta::Zero(); test.router()->OnBitrateUpdated(update, /*framerate*/ 0); } TEST(RtpVideoSenderTest, SimulcastSenderRegistersFrameTransformers) { rtc::scoped_refptr transformer = new rtc::RefCountedObject(); EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc1)); EXPECT_CALL(*transformer, RegisterTransformedFrameSinkCallback(_, kSsrc2)); RtpVideoSenderTestFixture test({kSsrc1, kSsrc2}, {kRtxSsrc1, kRtxSsrc2}, kPayloadType, {}, nullptr, transformer); EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc1)); EXPECT_CALL(*transformer, UnregisterTransformedFrameSinkCallback(kSsrc2)); } } // namespace webrtc