/* * Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #import #import #include "sdk/objc/native/src/objc_video_track_source.h" #import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h" #import "base/RTCVideoFrame.h" #import "base/RTCVideoFrameBuffer.h" #import "components/video_frame_buffer/RTCCVPixelBuffer.h" #import "frame_buffer_helpers.h" #include "api/scoped_refptr.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "media/base/fake_video_renderer.h" #include "rtc_base/ref_counted_object.h" #include "sdk/objc/native/api/video_frame.h" typedef void (^VideoSinkCallback)(RTC_OBJC_TYPE(RTCVideoFrame) *); namespace { class ObjCCallbackVideoSink : public rtc::VideoSinkInterface { public: ObjCCallbackVideoSink(VideoSinkCallback callback) : callback_(callback) {} void OnFrame(const webrtc::VideoFrame &frame) override { callback_(NativeToObjCVideoFrame(frame)); } private: VideoSinkCallback callback_; }; } // namespace @interface ObjCVideoTrackSourceTests : XCTestCase @end @implementation ObjCVideoTrackSourceTests { rtc::scoped_refptr _video_source; } - (void)setUp { _video_source = new rtc::RefCountedObject(); } - (void)tearDown { _video_source = NULL; } - (void)testOnCapturedFrameAdaptsFrame { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); XCTAssertEqual(video_renderer->num_rendered_frames(), 1); XCTAssertEqual(video_renderer->width(), 360); XCTAssertEqual(video_renderer->height(), 640); CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameAdaptsFrameWithAlignment { // Requesting to adapt 1280x720 to 912x514 gives 639x360 without alignment. The 639 causes issues // with some hardware encoders (e.g. HEVC) so in this test we verify that the alignment is set and // respected. CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); _video_source->OnOutputFormatRequest(912, 514, 30); _video_source->OnCapturedFrame(frame); XCTAssertEqual(video_renderer->num_rendered_frames(), 1); XCTAssertEqual(video_renderer->width(), 360); XCTAssertEqual(video_renderer->height(), 640); CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameAdaptationResultsInCommonResolutions { // Some of the most common resolutions used in the wild are 640x360, 480x270 and 320x180. // Make sure that we properly scale down to exactly these resolutions. CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; cricket::FakeVideoRenderer *video_renderer = new cricket::FakeVideoRenderer(); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(video_renderer, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); XCTAssertEqual(video_renderer->num_rendered_frames(), 1); XCTAssertEqual(video_renderer->width(), 360); XCTAssertEqual(video_renderer->height(), 640); _video_source->OnOutputFormatRequest(480, 270, 30); _video_source->OnCapturedFrame(frame); XCTAssertEqual(video_renderer->num_rendered_frames(), 2); XCTAssertEqual(video_renderer->width(), 270); XCTAssertEqual(video_renderer->height(), 480); _video_source->OnOutputFormatRequest(320, 180, 30); _video_source->OnCapturedFrame(frame); XCTAssertEqual(video_renderer->num_rendered_frames(), 3); XCTAssertEqual(video_renderer->width(), 180); XCTAssertEqual(video_renderer->height(), 320); CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameWithoutAdaptation { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 360, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(frame.width, outputFrame.width); XCTAssertEqual(frame.height, outputFrame.height); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(buffer.cropX, outputBuffer.cropX); XCTAssertEqual(buffer.cropY, outputBuffer.cropY); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameCVPixelBufferNeedsAdaptation { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 360); XCTAssertEqual(outputFrame.height, 640); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(outputBuffer.cropX, 0); XCTAssertEqual(outputBuffer.cropY, 0); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameCVPixelBufferNeedsCropping { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 360); XCTAssertEqual(outputFrame.height, 640); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(outputBuffer.cropX, 10); XCTAssertEqual(outputBuffer.cropY, 0); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFramePreAdaptedCVPixelBufferNeedsAdaptation { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); // Create a frame that's already adapted down. RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:640 adaptedHeight:360 cropWidth:720 cropHeight:1280 cropX:0 cropY:0]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 480); XCTAssertEqual(outputFrame.height, 270); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(outputBuffer.cropX, 0); XCTAssertEqual(outputBuffer.cropY, 0); XCTAssertEqual(outputBuffer.cropWidth, 640); XCTAssertEqual(outputBuffer.cropHeight, 360); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(480, 270, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFramePreCroppedCVPixelBufferNeedsCropping { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:370 adaptedHeight:640 cropWidth:370 cropHeight:640 cropX:10 cropY:0]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 360); XCTAssertEqual(outputFrame.height, 640); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(outputBuffer.cropX, 14); XCTAssertEqual(outputBuffer.cropY, 0); XCTAssertEqual(outputBuffer.cropWidth, 360); XCTAssertEqual(outputBuffer.cropHeight, 640); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameSmallerPreCroppedCVPixelBufferNeedsCropping { CVPixelBufferRef pixelBufferRef = NULL; CVPixelBufferCreate( NULL, 380, 640, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef); RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef adaptedWidth:300 adaptedHeight:640 cropWidth:300 cropHeight:640 cropX:40 cropY:0]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 300); XCTAssertEqual(outputFrame.height, 534); RTC_OBJC_TYPE(RTCCVPixelBuffer) *outputBuffer = outputFrame.buffer; XCTAssertEqual(outputBuffer.cropX, 40); XCTAssertEqual(outputBuffer.cropY, 52); XCTAssertEqual(outputBuffer.cropWidth, 300); XCTAssertEqual(outputBuffer.cropHeight, 534); XCTAssertEqual(buffer.pixelBuffer, outputBuffer.pixelBuffer); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; CVBufferRelease(pixelBufferRef); } - (void)testOnCapturedFrameI420BufferNeedsAdaptation { rtc::scoped_refptr i420Buffer = CreateI420Gradient(720, 1280); RTC_OBJC_TYPE(RTCI420Buffer) *buffer = [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 360); XCTAssertEqual(outputFrame.height, 640); RTC_OBJC_TYPE(RTCI420Buffer) *outputBuffer = (RTC_OBJC_TYPE(RTCI420Buffer) *)outputFrame.buffer; double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); XCTAssertEqual(psnr, webrtc::kPerfectPSNR); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; } - (void)testOnCapturedFrameI420BufferNeedsCropping { rtc::scoped_refptr i420Buffer = CreateI420Gradient(380, 640); RTC_OBJC_TYPE(RTCI420Buffer) *buffer = [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer]; RTC_OBJC_TYPE(RTCVideoFrame) *frame = [[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:buffer rotation:RTCVideoRotation_0 timeStampNs:0]; XCTestExpectation *callbackExpectation = [self expectationWithDescription:@"videoSinkCallback"]; ObjCCallbackVideoSink callback_video_sink(^void(RTC_OBJC_TYPE(RTCVideoFrame) * outputFrame) { XCTAssertEqual(outputFrame.width, 360); XCTAssertEqual(outputFrame.height, 640); RTC_OBJC_TYPE(RTCI420Buffer) *outputBuffer = (RTC_OBJC_TYPE(RTCI420Buffer) *)outputFrame.buffer; double psnr = I420PSNR(*[buffer nativeI420Buffer], *[outputBuffer nativeI420Buffer]); XCTAssertGreaterThanOrEqual(psnr, 40); [callbackExpectation fulfill]; }); const rtc::VideoSinkWants video_sink_wants; rtc::VideoSourceInterface *video_source_interface = _video_source; video_source_interface->AddOrUpdateSink(&callback_video_sink, video_sink_wants); _video_source->OnOutputFormatRequest(640, 360, 30); _video_source->OnCapturedFrame(frame); [self waitForExpectations:@[ callbackExpectation ] timeout:10.0]; } @end