/* * Copyright (C) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define LOG_TAG "EmulatedRequestProcessor" #define ATRACE_TAG ATRACE_TAG_CAMERA #include "EmulatedRequestProcessor.h" #include #include #include #include #include #include namespace android { using android::hardware::camera::common::V1_0::helper::HandleImporter; using google_camera_hal::ErrorCode; using google_camera_hal::HwlPipelineResult; using google_camera_hal::MessageType; using google_camera_hal::NotifyMessage; EmulatedRequestProcessor::EmulatedRequestProcessor( uint32_t camera_id, sp sensor, const HwlSessionCallback& session_callback) : camera_id_(camera_id), sensor_(sensor), session_callback_(session_callback), request_state_(std::make_unique(camera_id)) { ATRACE_CALL(); request_thread_ = std::thread([this] { this->RequestProcessorLoop(); }); importer_ = std::make_shared(); } EmulatedRequestProcessor::~EmulatedRequestProcessor() { ATRACE_CALL(); processor_done_ = true; request_thread_.join(); auto ret = sensor_->ShutDown(); if (ret != OK) { ALOGE("%s: Failed during sensor shutdown %s (%d)", __FUNCTION__, strerror(-ret), ret); } } status_t EmulatedRequestProcessor::ProcessPipelineRequests( uint32_t frame_number, std::vector& requests, const std::vector& pipelines, const DynamicStreamIdMapType& dynamic_stream_id_map, bool use_default_physical_camera) { ATRACE_CALL(); status_t res = OK; std::unique_lock lock(process_mutex_); for (auto& request : requests) { if (request.pipeline_id >= pipelines.size()) { ALOGE("%s: Pipeline request with invalid pipeline id: %u", __FUNCTION__, request.pipeline_id); return BAD_VALUE; } while (pending_requests_.size() > EmulatedSensor::kPipelineDepth) { auto result = request_condition_.wait_for( lock, std::chrono::nanoseconds( EmulatedSensor::kSupportedFrameDurationRange[1])); if (result == std::cv_status::timeout) { ALOGE("%s: Timed out waiting for a pending request slot", __FUNCTION__); return TIMED_OUT; } } res = request_state_->UpdateRequestForDynamicStreams( &request, pipelines, dynamic_stream_id_map, use_default_physical_camera); if (res != OK) { ALOGE("%s: Failed to update request for dynamic streams: %s(%d)", __FUNCTION__, strerror(-res), res); return res; } auto output_buffers = CreateSensorBuffers( frame_number, request.output_buffers, pipelines[request.pipeline_id].streams, request.pipeline_id, pipelines[request.pipeline_id].cb, /*override_width*/ 0, /*override_height*/ 0); if (output_buffers == nullptr) { return NO_MEMORY; } auto input_buffers = CreateSensorBuffers( frame_number, request.input_buffers, pipelines[request.pipeline_id].streams, request.pipeline_id, pipelines[request.pipeline_id].cb, request.input_width, request.input_height); pending_requests_.push( {.settings = HalCameraMetadata::Clone(request.settings.get()), .input_buffers = std::move(input_buffers), .output_buffers = std::move(output_buffers)}); } return OK; } std::unique_ptr EmulatedRequestProcessor::CreateSensorBuffers( uint32_t frame_number, const std::vector& buffers, const std::unordered_map& streams, uint32_t pipeline_id, HwlPipelineCallback cb, int32_t override_width, int32_t override_height) { if (buffers.empty()) { return nullptr; } std::vector requested_buffers; for (auto& buffer : buffers) { if (buffer.buffer != nullptr) { requested_buffers.push_back(buffer); continue; } if (session_callback_.request_stream_buffers != nullptr) { std::vector one_requested_buffer; status_t res = session_callback_.request_stream_buffers( buffer.stream_id, 1, &one_requested_buffer, frame_number); if (res != OK) { ALOGE("%s: request_stream_buffers failed: %s(%d)", __FUNCTION__, strerror(-res), res); continue; } if (one_requested_buffer.size() != 1 || one_requested_buffer[0].buffer == nullptr) { ALOGE("%s: request_stream_buffers failed to return a valid buffer", __FUNCTION__); continue; } requested_buffers.push_back(one_requested_buffer[0]); } } if (requested_buffers.size() < buffers.size()) { ALOGE( "%s: Failed to acquire all sensor buffers: %zu acquired, %zu requested", __FUNCTION__, requested_buffers.size(), buffers.size()); // This only happens for HAL buffer manager use case. if (session_callback_.return_stream_buffers != nullptr) { session_callback_.return_stream_buffers(requested_buffers); } return nullptr; } auto sensor_buffers = std::make_unique(); sensor_buffers->reserve(requested_buffers.size()); for (auto& buffer : requested_buffers) { auto sensor_buffer = CreateSensorBuffer( frame_number, streams.at(buffer.stream_id), pipeline_id, cb, buffer, override_width, override_height); if (sensor_buffer.get() != nullptr) { sensor_buffers->push_back(std::move(sensor_buffer)); } } return sensor_buffers; } void EmulatedRequestProcessor::NotifyFailedRequest(const PendingRequest& request) { if (request.output_buffers->at(0)->callback.notify != nullptr) { // Mark all output buffers for this request in order not to send // ERROR_BUFFER for them. for (auto& output_buffer : *(request.output_buffers)) { output_buffer->is_failed_request = true; } auto output_buffer = std::move(request.output_buffers->at(0)); NotifyMessage msg = { .type = MessageType::kError, .message.error = {.frame_number = output_buffer->frame_number, .error_stream_id = -1, .error_code = ErrorCode::kErrorRequest}}; output_buffer->callback.notify(output_buffer->pipeline_id, msg); } } status_t EmulatedRequestProcessor::Flush() { std::lock_guard lock(process_mutex_); // First flush in-flight requests auto ret = sensor_->Flush(); // Then the rest of the pending requests while (!pending_requests_.empty()) { const auto& request = pending_requests_.front(); NotifyFailedRequest(request); pending_requests_.pop(); } return ret; } status_t EmulatedRequestProcessor::GetBufferSizeAndStride( const EmulatedStream& stream, buffer_handle_t buffer, uint32_t* size /*out*/, uint32_t* stride /*out*/) { if (size == nullptr) { return BAD_VALUE; } switch (stream.override_format) { case HAL_PIXEL_FORMAT_RGB_888: *stride = stream.width * 3; *size = (*stride) * stream.height; break; case HAL_PIXEL_FORMAT_RGBA_8888: *stride = stream.width * 4; *size = (*stride) * stream.height; break; case HAL_PIXEL_FORMAT_Y16: if (stream.override_data_space == HAL_DATASPACE_DEPTH) { *stride = AlignTo(AlignTo(stream.width, 2) * 2, 16); *size = (*stride) * AlignTo(stream.height, 2); } else { return BAD_VALUE; } break; case HAL_PIXEL_FORMAT_BLOB: if (stream.override_data_space == HAL_DATASPACE_V0_JFIF) { *size = stream.buffer_size; *stride = *size; } else { return BAD_VALUE; } break; case HAL_PIXEL_FORMAT_RAW16: if (importer_->getMonoPlanarStrideBytes(buffer, stride) != NO_ERROR) { *stride = stream.width * 2; } *size = (*stride) * stream.height; break; default: return BAD_VALUE; } return OK; } status_t EmulatedRequestProcessor::LockSensorBuffer( const EmulatedStream& stream, buffer_handle_t buffer, int32_t width, int32_t height, SensorBuffer* sensor_buffer /*out*/) { if (sensor_buffer == nullptr) { return BAD_VALUE; } auto usage = GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN; bool isYUV_420_888 = stream.override_format == HAL_PIXEL_FORMAT_YCBCR_420_888; bool isP010 = static_cast( stream.override_format) == HAL_PIXEL_FORMAT_YCBCR_P010; if ((isYUV_420_888) || (isP010)) { IMapper::Rect map_rect = {0, 0, width, height}; auto yuv_layout = importer_->lockYCbCr(buffer, usage, map_rect); if ((yuv_layout.y != nullptr) && (yuv_layout.cb != nullptr) && (yuv_layout.cr != nullptr)) { sensor_buffer->plane.img_y_crcb.img_y = static_cast(yuv_layout.y); sensor_buffer->plane.img_y_crcb.img_cb = static_cast(yuv_layout.cb); sensor_buffer->plane.img_y_crcb.img_cr = static_cast(yuv_layout.cr); sensor_buffer->plane.img_y_crcb.y_stride = yuv_layout.yStride; sensor_buffer->plane.img_y_crcb.cbcr_stride = yuv_layout.cStride; sensor_buffer->plane.img_y_crcb.cbcr_step = yuv_layout.chromaStep; if (isYUV_420_888 && (yuv_layout.chromaStep == 2) && std::abs(sensor_buffer->plane.img_y_crcb.img_cb - sensor_buffer->plane.img_y_crcb.img_cr) != 1) { ALOGE("%s: Unsupported YUV layout, chroma step: %u U/V plane delta: %u", __FUNCTION__, yuv_layout.chromaStep, static_cast( std::abs(sensor_buffer->plane.img_y_crcb.img_cb - sensor_buffer->plane.img_y_crcb.img_cr))); return BAD_VALUE; } sensor_buffer->plane.img_y_crcb.bytesPerPixel = isP010 ? 2 : 1; } else { ALOGE("%s: Failed to lock output buffer!", __FUNCTION__); return BAD_VALUE; } } else { uint32_t buffer_size = 0, stride = 0; auto ret = GetBufferSizeAndStride(stream, buffer, &buffer_size, &stride); if (ret != OK) { ALOGE("%s: Unsupported pixel format: 0x%x", __FUNCTION__, stream.override_format); return BAD_VALUE; } if (stream.override_format == HAL_PIXEL_FORMAT_BLOB) { sensor_buffer->plane.img.img = static_cast(importer_->lock(buffer, usage, buffer_size)); } else { IMapper::Rect region{0, 0, width, height}; sensor_buffer->plane.img.img = static_cast(importer_->lock(buffer, usage, region)); } if (sensor_buffer->plane.img.img == nullptr) { ALOGE("%s: Failed to lock output buffer!", __FUNCTION__); return BAD_VALUE; } sensor_buffer->plane.img.stride_in_bytes = stride; sensor_buffer->plane.img.buffer_size = buffer_size; } return OK; } std::unique_ptr EmulatedRequestProcessor::CreateSensorBuffer( uint32_t frame_number, const EmulatedStream& emulated_stream, uint32_t pipeline_id, HwlPipelineCallback callback, StreamBuffer stream_buffer, int32_t override_width, int32_t override_height) { auto buffer = std::make_unique(importer_); auto stream = emulated_stream; // Make sure input stream formats are correctly mapped here if (stream.is_input) { stream.override_format = EmulatedSensor::OverrideFormat(stream.override_format); } if (override_width > 0 && override_height > 0) { buffer->width = override_width; buffer->height = override_height; } else { buffer->width = stream.width; buffer->height = stream.height; } buffer->format = static_cast(stream.override_format); buffer->dataSpace = stream.override_data_space; buffer->stream_buffer = stream_buffer; buffer->pipeline_id = pipeline_id; buffer->callback = callback; buffer->frame_number = frame_number; buffer->camera_id = emulated_stream.is_physical_camera_stream ? emulated_stream.physical_camera_id : camera_id_; buffer->is_input = stream.is_input; // In case buffer processing is successful, flip this flag accordingly buffer->stream_buffer.status = BufferStatus::kError; if (buffer->stream_buffer.buffer != nullptr) { auto ret = LockSensorBuffer(stream, buffer->stream_buffer.buffer, buffer->width, buffer->height, buffer.get()); if (ret != OK) { buffer.release(); buffer = nullptr; } } if ((buffer.get() != nullptr) && (stream_buffer.acquire_fence != nullptr)) { auto fence_status = importer_->importFence(stream_buffer.acquire_fence, buffer->acquire_fence_fd); if (!fence_status) { ALOGE("%s: Failed importing acquire fence!", __FUNCTION__); buffer.release(); buffer = nullptr; } } return buffer; } std::unique_ptr EmulatedRequestProcessor::AcquireBuffers( Buffers* buffers) { if ((buffers == nullptr) || (buffers->empty())) { return nullptr; } auto acquired_buffers = std::make_unique(); acquired_buffers->reserve(buffers->size()); auto output_buffer = buffers->begin(); while (output_buffer != buffers->end()) { status_t ret = OK; if ((*output_buffer)->acquire_fence_fd >= 0) { ret = sync_wait((*output_buffer)->acquire_fence_fd, ns2ms(EmulatedSensor::kSupportedFrameDurationRange[1])); if (ret != OK) { ALOGE("%s: Fence sync failed: %s, (%d)", __FUNCTION__, strerror(-ret), ret); } } if (ret == OK) { acquired_buffers->push_back(std::move(*output_buffer)); } output_buffer = buffers->erase(output_buffer); } return acquired_buffers; } void EmulatedRequestProcessor::RequestProcessorLoop() { ATRACE_CALL(); bool vsync_status_ = true; while (!processor_done_ && vsync_status_) { { std::lock_guard lock(process_mutex_); if (!pending_requests_.empty()) { status_t ret; const auto& request = pending_requests_.front(); auto frame_number = request.output_buffers->at(0)->frame_number; auto notify_callback = request.output_buffers->at(0)->callback; auto pipeline_id = request.output_buffers->at(0)->pipeline_id; auto output_buffers = AcquireBuffers(request.output_buffers.get()); auto input_buffers = AcquireBuffers(request.input_buffers.get()); if (!output_buffers->empty()) { std::unique_ptr logical_settings = std::make_unique(); std::unique_ptr> physical_camera_output_ids = std::make_unique>(); for (const auto& it : *output_buffers) { if (it->camera_id != camera_id_) { physical_camera_output_ids->emplace(it->camera_id); } } // Repeating requests usually include valid settings only during the // initial call. Afterwards an invalid settings pointer means that // there are no changes in the parameters and Hal should re-use the // last valid values. // TODO: Add support for individual physical camera requests. if (request.settings.get() != nullptr) { ret = request_state_->InitializeLogicalSettings( HalCameraMetadata::Clone(request.settings.get()), std::move(physical_camera_output_ids), logical_settings.get()); last_settings_ = HalCameraMetadata::Clone(request.settings.get()); } else { ret = request_state_->InitializeLogicalSettings( HalCameraMetadata::Clone(last_settings_.get()), std::move(physical_camera_output_ids), logical_settings.get()); } if (ret == OK) { auto result = request_state_->InitializeLogicalResult(pipeline_id, frame_number); sensor_->SetCurrentRequest( std::move(logical_settings), std::move(result), std::move(input_buffers), std::move(output_buffers)); } else { NotifyMessage msg{.type = MessageType::kError, .message.error = { .frame_number = frame_number, .error_stream_id = -1, .error_code = ErrorCode::kErrorResult, }}; notify_callback.notify(pipeline_id, msg); } } else { // No further processing is needed, just fail the result which will // complete this request. NotifyMessage msg{.type = MessageType::kError, .message.error = { .frame_number = frame_number, .error_stream_id = -1, .error_code = ErrorCode::kErrorResult, }}; notify_callback.notify(pipeline_id, msg); } pending_requests_.pop(); request_condition_.notify_one(); } } vsync_status_ = sensor_->WaitForVSync(EmulatedSensor::kSupportedFrameDurationRange[1]); } } status_t EmulatedRequestProcessor::Initialize( std::unique_ptr static_meta, PhysicalDeviceMapPtr physical_devices) { std::lock_guard lock(process_mutex_); return request_state_->Initialize(std::move(static_meta), std::move(physical_devices)); } void EmulatedRequestProcessor::SetSessionCallback( const HwlSessionCallback& hwl_session_callback) { std::lock_guard lock(process_mutex_); session_callback_ = hwl_session_callback; } status_t EmulatedRequestProcessor::GetDefaultRequest( RequestTemplate type, std::unique_ptr* default_settings) { std::lock_guard lock(process_mutex_); return request_state_->GetDefaultRequest(type, default_settings); } } // namespace android