You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
760 lines
30 KiB
760 lines
30 KiB
// Copyright (C) 2019 The Android Open Source Project
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#include "host-common/MediaH264DecoderVideoToolBox.h"
|
|
|
|
#include "host-common/H264NaluParser.h"
|
|
|
|
#include <VideoToolbox/VideoToolbox.h>
|
|
|
|
#include <cstdint>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
|
|
#ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
|
|
#define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
|
|
#endif
|
|
|
|
#define MEDIA_H264_DEBUG 0
|
|
|
|
#if MEDIA_H264_DEBUG
|
|
#define H264_DPRINT(fmt,...) fprintf(stderr, "h264-videotoolbox-dec: %s:%d " fmt "\n", __func__, __LINE__, ##__VA_ARGS__);
|
|
#else
|
|
#define H264_DPRINT(fmt,...)
|
|
#endif
|
|
|
|
|
|
namespace android {
|
|
namespace emulation {
|
|
|
|
using InitContextParam = H264PingInfoParser::InitContextParam;
|
|
using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
|
|
using ResetParam = H264PingInfoParser::ResetParam;
|
|
using GetImageParam = H264PingInfoParser::GetImageParam;
|
|
using H264NaluType = H264NaluParser::H264NaluType;
|
|
|
|
MediaH264DecoderVideoToolBox::MediaH264DecoderVideoToolBox(
|
|
uint64_t id,
|
|
H264PingInfoParser parser)
|
|
: mId(id), mParser(parser) {
|
|
H264_DPRINT("created MediaH264DecoderVideoToolBox %p", this);
|
|
}
|
|
|
|
MediaH264DecoderPlugin* MediaH264DecoderVideoToolBox::clone() {
|
|
return new MediaH264DecoderVideoToolBox(mId, mParser);
|
|
}
|
|
|
|
MediaH264DecoderVideoToolBox::~MediaH264DecoderVideoToolBox() {
|
|
destroyH264Context();
|
|
}
|
|
|
|
// static
|
|
void MediaH264DecoderVideoToolBox::videoToolboxDecompressCallback(void* opaque,
|
|
void* sourceFrameRefCon,
|
|
OSStatus status,
|
|
VTDecodeInfoFlags flags,
|
|
CVImageBufferRef image_buffer,
|
|
CMTime pts,
|
|
CMTime duration) {
|
|
H264_DPRINT("%s", __func__);
|
|
auto ptr = static_cast<MediaH264DecoderVideoToolBox*>(opaque);
|
|
|
|
if (ptr->mDecodedFrame) {
|
|
CVPixelBufferRelease(ptr->mDecodedFrame);
|
|
ptr->mDecodedFrame = nullptr;
|
|
}
|
|
|
|
if (!image_buffer) {
|
|
H264_DPRINT("%s: output image buffer is null", __func__);
|
|
return;
|
|
}
|
|
|
|
ptr->mOutputPts = pts.value;
|
|
ptr->mDecodedFrame = CVPixelBufferRetain(image_buffer);
|
|
// Image is ready to be comsumed
|
|
ptr->copyFrame();
|
|
ptr->mImageReady = true;
|
|
H264_DPRINT("Got decoded frame");
|
|
}
|
|
|
|
// static
|
|
CFDictionaryRef MediaH264DecoderVideoToolBox::createOutputBufferAttributes(int width,
|
|
int height,
|
|
OSType pix_fmt) {
|
|
CFMutableDictionaryRef buffer_attributes;
|
|
CFMutableDictionaryRef io_surface_properties;
|
|
CFNumberRef cv_pix_fmt;
|
|
CFNumberRef w;
|
|
CFNumberRef h;
|
|
|
|
w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
|
|
h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
|
|
cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
|
|
|
|
buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
|
|
4,
|
|
&kCFTypeDictionaryKeyCallBacks,
|
|
&kCFTypeDictionaryValueCallBacks);
|
|
io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
|
|
0,
|
|
&kCFTypeDictionaryKeyCallBacks,
|
|
&kCFTypeDictionaryValueCallBacks);
|
|
|
|
if (pix_fmt) {
|
|
CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
|
|
}
|
|
CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
|
|
CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
|
|
CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
|
|
// Not sure if this will work becuase we are passing the pixel buffer back into the guest
|
|
CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
|
|
|
|
CFRelease(io_surface_properties);
|
|
CFRelease(cv_pix_fmt);
|
|
CFRelease(w);
|
|
CFRelease(h);
|
|
|
|
return buffer_attributes;
|
|
}
|
|
|
|
// static
|
|
CMSampleBufferRef MediaH264DecoderVideoToolBox::createSampleBuffer(CMFormatDescriptionRef fmtDesc,
|
|
void* buffer,
|
|
size_t sz) {
|
|
OSStatus status;
|
|
CMBlockBufferRef blockBuf = nullptr;
|
|
CMSampleBufferRef sampleBuf = nullptr;
|
|
|
|
status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, // structureAllocator
|
|
buffer, // memoryBlock
|
|
sz, // blockLength
|
|
kCFAllocatorNull, // blockAllocator
|
|
NULL, // customBlockSource
|
|
0, // offsetToData
|
|
sz, // dataLength
|
|
0, // flags
|
|
&blockBuf);
|
|
|
|
if (!status) {
|
|
status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
|
|
blockBuf, // dataBuffer
|
|
TRUE, // dataReady
|
|
0, // makeDataReadyCallback
|
|
0, // makeDataReadyRefCon
|
|
fmtDesc, // formatDescription
|
|
1, // numSamples
|
|
0, // numSampleTimingEntries
|
|
NULL, // sampleTimingArray
|
|
0, // numSampleSizeEntries
|
|
NULL, // sampleSizeArray
|
|
&sampleBuf);
|
|
}
|
|
|
|
if (blockBuf) {
|
|
CFRelease(blockBuf);
|
|
}
|
|
|
|
return sampleBuf;
|
|
}
|
|
|
|
// static
|
|
OSType MediaH264DecoderVideoToolBox::toNativePixelFormat(PixelFormat pixFmt) {
|
|
switch (pixFmt) {
|
|
case PixelFormat::YUV420P:
|
|
return kCVPixelFormatType_420YpCbCr8Planar;
|
|
case PixelFormat::UYVY422:
|
|
return kCVPixelFormatType_422YpCbCr8;
|
|
case PixelFormat::BGRA8888:
|
|
return kCVPixelFormatType_32BGRA;
|
|
default:
|
|
H264_DPRINT("Unsupported VideoToolbox pixel format");
|
|
return '0000';
|
|
}
|
|
}
|
|
|
|
// static
|
|
void* MediaH264DecoderVideoToolBox::getReturnAddress(void* ptr) {
|
|
uint8_t* xptr = (uint8_t*)ptr;
|
|
void* pint = (void*)(xptr + 256);
|
|
return pint;
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::createCMFormatDescription() {
|
|
uint8_t* parameterSets[2] = {mSPS.data(), mPPS.data()};
|
|
size_t parameterSetSizes[2] = {mSPS.size(), mPPS.size()};
|
|
|
|
if (mCmFmtDesc) {
|
|
CFRelease(mCmFmtDesc);
|
|
mCmFmtDesc = nullptr;
|
|
}
|
|
|
|
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
|
|
kCFAllocatorDefault,
|
|
2,
|
|
(const uint8_t *const*)parameterSets,
|
|
parameterSetSizes,
|
|
4,
|
|
&mCmFmtDesc);
|
|
|
|
if (status == noErr) {
|
|
H264_DPRINT("Created CMFormatDescription from SPS/PPS sets");
|
|
} else {
|
|
H264_DPRINT("Unable to create CMFormatDescription (%d)\n", (int)status);
|
|
}
|
|
}
|
|
|
|
CFDataRef MediaH264DecoderVideoToolBox::createVTDecoderConfig() {
|
|
CFDataRef data = nullptr;
|
|
return data;
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::initH264Context(void* ptr) {
|
|
InitContextParam param{};
|
|
mParser.parseInitContextParams(ptr, param);
|
|
initH264ContextInternal(param.width, param.height, param.outputWidth,
|
|
param.outputHeight, param.outputPixelFormat);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::initH264ContextInternal(
|
|
unsigned int width,
|
|
unsigned int height,
|
|
unsigned int outWidth,
|
|
unsigned int outHeight,
|
|
PixelFormat outPixFmt) {
|
|
H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)",
|
|
__func__, width, height, outWidth, outHeight, (uint8_t)outPixFmt);
|
|
mWidth = width;
|
|
mHeight = height;
|
|
mOutputWidth = outWidth;
|
|
mOutputHeight = outHeight;
|
|
mOutPixFmt = outPixFmt;
|
|
mOutBufferSize = outWidth * outHeight * 3 / 2;
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::reset(void* ptr) {
|
|
destroyH264Context();
|
|
ResetParam param{};
|
|
mParser.parseResetParams(ptr, param);
|
|
initH264ContextInternal(param.width, param.height, param.outputWidth,
|
|
param.outputHeight, param.outputPixelFormat);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::destroyH264Context() {
|
|
H264_DPRINT("%s", __func__);
|
|
if (mDecoderSession) {
|
|
VTDecompressionSessionInvalidate(mDecoderSession);
|
|
CFRelease(mDecoderSession);
|
|
mDecoderSession = nullptr;
|
|
}
|
|
if (mCmFmtDesc) {
|
|
CFRelease(mCmFmtDesc);
|
|
mCmFmtDesc = nullptr;
|
|
}
|
|
if (mDecodedFrame) {
|
|
CVPixelBufferRelease(mDecodedFrame);
|
|
mDecodedFrame = nullptr;
|
|
}
|
|
}
|
|
|
|
static void dumpBytes(const uint8_t* img, size_t szBytes, bool all = false) {
|
|
#if MEDIA_H264_DEBUG
|
|
printf("data=");
|
|
size_t numBytes = szBytes;
|
|
if (!all) {
|
|
numBytes = 32;
|
|
}
|
|
|
|
for (size_t i = 0; i < (numBytes > szBytes ? szBytes : numBytes); ++i) {
|
|
if (i % 8 == 0) {
|
|
printf("\n");
|
|
}
|
|
printf("0x%02x ", img[i]);
|
|
}
|
|
printf("\n");
|
|
#endif
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::decodeFrame(void* ptr) {
|
|
DecodeFrameParam param{};
|
|
mParser.parseDecodeFrameParams(ptr, param);
|
|
|
|
const uint8_t* frame = param.pData;
|
|
size_t szBytes = param.size;
|
|
uint64_t pts = param.pts;
|
|
|
|
decodeFrameInternal(param.pConsumedBytes, param.pDecoderErrorCode, frame, szBytes, pts, 0);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::oneShotDecode(std::vector<uint8_t> & data, uint64_t pts) {
|
|
//TODO: check if data has more than one Nalu
|
|
decodeFrameInternal(nullptr, nullptr, data.data(), data.size(), pts, 0);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::decodeFrameInternal(size_t* pRetSzBytes, int32_t* pRetErr, const uint8_t* frame, size_t szBytes, uint64_t pts, size_t consumedSzBytes) {
|
|
// IMPORTANT: There is an assumption that each |frame| we get from the guest are contain
|
|
// complete NALUs. Usually, an H.264 bitstream would be continuously fed to us, but in this case,
|
|
// it seems that the Android frameworks passes us complete NALUs, and may be more than one NALU at
|
|
// a time.
|
|
//
|
|
// Let's only process one NALU per decodeFrame() call because, as soon as we receive a PPS NALU,
|
|
// we need to let the OMX plugin know to reset it's state because we are also recreating our
|
|
// decoder context.
|
|
H264_DPRINT("%s(frame=%p, sz=%zu)", __func__, frame, szBytes);
|
|
Err h264Err = Err::NoErr;
|
|
|
|
const uint8_t* currentNalu = H264NaluParser::getNextStartCodeHeader(frame, szBytes);
|
|
if (currentNalu == nullptr) {
|
|
H264_DPRINT("No start code header found in this frame");
|
|
h264Err = Err::NoDecodedFrame;
|
|
// TODO: return the error code and num bytes processed, szBytes.
|
|
if (pRetSzBytes) *pRetSzBytes = szBytes;
|
|
if (pRetErr) *pRetErr = (int32_t)h264Err;
|
|
return;
|
|
}
|
|
const uint8_t* nextNalu = nullptr;
|
|
size_t remaining = szBytes - (currentNalu - frame);
|
|
|
|
// Figure out the size of |currentNalu|.
|
|
size_t currentNaluSize = remaining;
|
|
// 3 is the minimum size of the start code header (3 or 4 bytes).
|
|
dumpBytes(currentNalu, currentNaluSize);
|
|
nextNalu = H264NaluParser::getNextStartCodeHeader(currentNalu + 3, remaining - 3);
|
|
if (nextNalu != nullptr) {
|
|
currentNaluSize = nextNalu - currentNalu;
|
|
}
|
|
|
|
// |data| is currentNalu, but with the start code header discarded.
|
|
uint8_t* data = nullptr;
|
|
H264NaluType naluType = H264NaluParser::getFrameNaluType(currentNalu, currentNaluSize, &data);
|
|
size_t dataSize = currentNaluSize - (data - currentNalu);
|
|
const std::string naluTypeStr = H264NaluParser::naluTypeToString(naluType);
|
|
H264_DPRINT("Got frame type=%u (%s)", (uint8_t)naluType, naluTypeStr.c_str());
|
|
|
|
// We can't do anything until we set up a CMFormatDescription from a set of SPS and PPS NALUs.
|
|
// So just discard the NALU.
|
|
if (naluType != H264NaluType::SPS && naluType != H264NaluType::PPS &&
|
|
mCmFmtDesc == nullptr) {
|
|
H264_DPRINT("CMFormatDescription not set up yet. Need SPS/PPS frames.");
|
|
h264Err = Err::NALUIgnored;
|
|
if (pRetSzBytes) *pRetSzBytes = currentNaluSize;
|
|
if (pRetErr) *pRetErr = (int32_t)h264Err;
|
|
return;
|
|
}
|
|
|
|
switch (naluType) {
|
|
case H264NaluType::SPS:
|
|
// We should be getting a PPS frame on the next decodeFrame(). Once we have
|
|
// both sps and pps, we can create/recreate the decoder session.
|
|
// Don't include the start code header when we copy the sps/pps.
|
|
mSPS.assign(data, data + dataSize);
|
|
if (!mIsLoadingFromSnapshot) {
|
|
mSnapshotState = SnapshotState{};
|
|
std::vector<uint8_t> vec;
|
|
vec.assign(currentNalu, currentNalu + currentNaluSize);
|
|
mSnapshotState.saveSps(vec);
|
|
}
|
|
break;
|
|
case H264NaluType::PPS:
|
|
mPPS.assign(data, data + dataSize);
|
|
createCMFormatDescription();
|
|
// TODO: We will need to recreate the decompression session whenever we get a
|
|
// resolution change.
|
|
if (mDecoderSession != nullptr) {
|
|
H264_DPRINT("Decoder session is restarting");
|
|
//h264Err = Err::DecoderRestarted;
|
|
}
|
|
if (!mIsLoadingFromSnapshot) {
|
|
std::vector<uint8_t> vec;
|
|
vec.assign(currentNalu, currentNalu + currentNaluSize);
|
|
mSnapshotState.savePps(vec);
|
|
mSnapshotState.savedPackets.clear();
|
|
}
|
|
recreateDecompressionSession();
|
|
break;
|
|
case H264NaluType::SEI:
|
|
// dumpBytes(nextNalu, remaining, true);
|
|
// In some cases, after the SPS and PPS NALUs are emitted, we'll get a frame that
|
|
// contains both an SEI NALU and a CodedSliceIDR NALU.
|
|
handleSEIFrame(currentNalu, currentNaluSize);
|
|
break;
|
|
case H264NaluType::CodedSliceIDR:
|
|
handleIDRFrame(currentNalu, currentNaluSize, pts);
|
|
if (!mIsLoadingFromSnapshot) {
|
|
H264_DPRINT("disacarding previously saved frames %d", (int)mSnapshotState.savedPackets.size());
|
|
mSnapshotState.savedPackets.clear();
|
|
mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
|
|
}
|
|
break;
|
|
case H264NaluType::CodedSliceNonIDR:
|
|
handleNonIDRFrame(currentNalu, currentNaluSize, pts);
|
|
if (!mIsLoadingFromSnapshot) {
|
|
mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
|
|
}
|
|
break;
|
|
default:
|
|
H264_DPRINT("Support for nalu_type=%u not implemented", (uint8_t)naluType);
|
|
break;
|
|
}
|
|
|
|
remaining -= currentNaluSize;
|
|
currentNalu = nextNalu;
|
|
|
|
// return two things: the error code and the number of bytes we processed.
|
|
if (pRetSzBytes) *pRetSzBytes = currentNaluSize + consumedSzBytes;
|
|
if (pRetErr) *pRetErr = (int32_t)h264Err;
|
|
|
|
// disable recursive decoding due to the possibility of session creation failure
|
|
// keep it simple for now
|
|
//if (currentNalu) {
|
|
// decodeFrameInternal(ptr, currentNalu, remaining, pts, consumedSzBytes + currentNaluSize);
|
|
//}
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::handleIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
|
|
H264_DPRINT("Got IDR frame (sz=%zu)", szBytes);
|
|
uint8_t* fptr = const_cast<uint8_t*>(ptr);
|
|
|
|
// We can assume fptr has a valid start code header because it has already
|
|
// gone through validation in H264NaluParser.
|
|
uint8_t startHeaderSz = fptr[2] == 1 ? 3 : 4;
|
|
uint32_t dataSz = szBytes - startHeaderSz;
|
|
std::unique_ptr<uint8_t> idr(new uint8_t[dataSz + 4]);
|
|
uint32_t dataSzNl = htonl(dataSz);
|
|
|
|
// AVCC format requires us to replace the start code header on this NALU
|
|
// with the size of the data. Start code is either 0x000001 or 0x00000001.
|
|
// The size needs to be the first four bytes in network byte order.
|
|
memcpy(idr.get(), &dataSzNl, 4);
|
|
memcpy(idr.get() + 4, ptr + startHeaderSz, dataSz);
|
|
|
|
CMSampleBufferRef sampleBuf = nullptr;
|
|
sampleBuf = createSampleBuffer(mCmFmtDesc, (void*)idr.get(), dataSz + 4);
|
|
if (!sampleBuf) {
|
|
H264_DPRINT("%s: Failed to create CMSampleBufferRef", __func__);
|
|
return;
|
|
}
|
|
|
|
CMSampleBufferSetOutputPresentationTimeStamp(sampleBuf, CMTimeMake(pts, 1));
|
|
|
|
OSStatus status;
|
|
status = VTDecompressionSessionDecodeFrame(mDecoderSession,
|
|
sampleBuf,
|
|
0, // decodeFlags
|
|
NULL, // sourceFrameRefCon
|
|
0); // infoFlagsOut
|
|
|
|
if (status == noErr) {
|
|
// TODO: this call blocks until the frame has been decoded. Perhaps it will be
|
|
// more efficient to signal the guest when the frame is ready to be read instead.
|
|
status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
|
|
mIsInFlush = false;
|
|
} else {
|
|
H264_DPRINT("%s: Failed to decompress frame (err=%d)", __func__, status);
|
|
}
|
|
|
|
CFRelease(sampleBuf);
|
|
H264_DPRINT("Success decoding IDR frame");
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::handleNonIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
|
|
// Same as handling an IDR frame
|
|
handleIDRFrame(ptr, szBytes, pts);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::handleSEIFrame(const uint8_t* ptr, size_t szBytes) {
|
|
H264_DPRINT("NOT IMPLEMENTED");
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::flush(void* ptr) {
|
|
H264_DPRINT("%s: NOT IMPLEMENTED", __func__);
|
|
mIsInFlush = true;
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::copyFrame() {
|
|
if (mIsLoadingFromSnapshot) return;
|
|
|
|
int imgWidth = CVPixelBufferGetWidth(mDecodedFrame);
|
|
int imgHeight = CVPixelBufferGetHeight(mDecodedFrame);
|
|
int imageSize = CVPixelBufferGetDataSize(mDecodedFrame);
|
|
int stride = CVPixelBufferGetBytesPerRow(mDecodedFrame);
|
|
|
|
mOutputWidth = imgWidth;
|
|
mOutputHeight = imgHeight;
|
|
mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
|
|
|
|
H264_DPRINT("copying size=%d dimension=[%dx%d] stride=%d", imageSize, imgWidth, imgHeight, stride);
|
|
|
|
// Copies the image data to the guest.
|
|
mSavedDecodedFrame.resize(imgWidth * imgHeight * 3 / 2);
|
|
uint8_t* dst = mSavedDecodedFrame.data();
|
|
|
|
CVPixelBufferLockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
|
|
if (CVPixelBufferIsPlanar(mDecodedFrame)) {
|
|
imageSize = 0; // add up the size from the planes
|
|
int planes = CVPixelBufferGetPlaneCount(mDecodedFrame);
|
|
for (int i = 0; i < planes; ++i) {
|
|
void* planeData = CVPixelBufferGetBaseAddressOfPlane(mDecodedFrame, i);
|
|
int linesize = CVPixelBufferGetBytesPerRowOfPlane(mDecodedFrame, i);
|
|
int planeWidth = CVPixelBufferGetWidthOfPlane(mDecodedFrame, i);
|
|
int planeHeight = CVPixelBufferGetHeightOfPlane(mDecodedFrame, i);
|
|
H264_DPRINT("plane=%d data=%p linesize=%d pwidth=%d pheight=%d", i, planeData, linesize, planeWidth, planeHeight);
|
|
// For kCVPixelFormatType_420YpCbCr8Planar, plane 0 is Y, UV planes are 1 and 2
|
|
if (planeWidth != imgWidth && planeWidth != imgWidth / 2) {
|
|
H264_DPRINT("ERROR: Unable to determine YUV420 plane type");
|
|
continue;
|
|
}
|
|
|
|
// Sometimes the buffer stride can be longer than the actual data width. This means that
|
|
// the extra bytes are just padding and need to be discarded.
|
|
if (linesize <= planeWidth) {
|
|
int sz = planeHeight * planeWidth;
|
|
imageSize += sz;
|
|
memcpy(dst, planeData, sz);
|
|
dst += sz;
|
|
} else {
|
|
// Need to copy line by line
|
|
int sz = planeWidth;
|
|
for (int j = 0; j < planeHeight; ++j) {
|
|
uint8_t* ptr = (uint8_t*)planeData;
|
|
ptr += linesize * j;
|
|
memcpy(dst, ptr, sz);
|
|
imageSize += sz;
|
|
dst += sz;
|
|
}
|
|
}
|
|
}
|
|
if (imageSize != mOutBufferSize) {
|
|
H264_DPRINT("ERROR: Total size of planes not same as guestSz (guestSz=%u, imageSize=%d)", mOutBufferSize, imageSize);
|
|
}
|
|
} else {
|
|
if (imageSize > mOutBufferSize) {
|
|
H264_DPRINT("Buffer size mismatch (guestSz=%u, imageSize=%d). Using guestSz instead.", mOutBufferSize, imageSize);
|
|
imageSize = mOutBufferSize;
|
|
}
|
|
|
|
// IMPORTANT: mDecodedFrame must be locked before accessing the contents with CPU
|
|
void* data = CVPixelBufferGetBaseAddress(mDecodedFrame);
|
|
memcpy(dst, data, imageSize);
|
|
}
|
|
CVPixelBufferUnlockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::getImage(void* ptr) {
|
|
// return parameters:
|
|
// 1) either size of the image (> 0) or error code (<= 0).
|
|
// 2) image width
|
|
// 3) image height
|
|
GetImageParam param{};
|
|
mParser.parseGetImageParams(ptr, param);
|
|
|
|
int* retErr = param.pDecoderErrorCode;
|
|
uint32_t* retWidth = param.pRetWidth;
|
|
uint32_t* retHeight = param.pRetHeight;
|
|
uint64_t* retPts = param.pRetPts;
|
|
uint32_t* retColorPrimaries = param.pRetColorPrimaries;
|
|
uint32_t* retColorRange = param.pRetColorRange;
|
|
uint32_t* retColorTransfer = param.pRetColorTransfer;
|
|
uint32_t* retColorSpace = param.pRetColorSpace;
|
|
|
|
if (!mDecodedFrame) {
|
|
H264_DPRINT("%s: frame is null", __func__);
|
|
*retErr = static_cast<int>(Err::NoDecodedFrame);
|
|
return;
|
|
}
|
|
if (!mImageReady) {
|
|
bool hasMoreFrames = false;
|
|
if (mIsInFlush) {
|
|
OSStatus status = noErr;
|
|
status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
|
|
if (status == noErr) {
|
|
hasMoreFrames = mImageReady;
|
|
if (hasMoreFrames) {
|
|
H264_DPRINT("%s: got frame in flush mode", __func__);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!hasMoreFrames) {
|
|
H264_DPRINT("%s: no new frame yet", __func__);
|
|
*retErr = static_cast<int>(Err::NoDecodedFrame);
|
|
return;
|
|
}
|
|
}
|
|
|
|
*retWidth = mOutputWidth;
|
|
*retHeight = mOutputHeight;
|
|
|
|
if (mParser.version() == 200 && param.hostColorBufferId >= 0) {
|
|
mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
|
|
mOutputWidth, mOutputHeight,
|
|
mSavedDecodedFrame.data());
|
|
} else {
|
|
memcpy(param.pDecodedFrame, mSavedDecodedFrame.data(), mSavedDecodedFrame.size());;
|
|
}
|
|
|
|
*retErr = mSavedDecodedFrame.size();
|
|
|
|
*retPts = mOutputPts;
|
|
|
|
H264_DPRINT("Copying completed pts %lld", (long long)mOutputPts);
|
|
mImageReady = false;
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::recreateDecompressionSession() {
|
|
if (mCmFmtDesc == nullptr) {
|
|
H264_DPRINT("CMFormatDescription not created. Need sps and pps NALUs.");
|
|
return;
|
|
}
|
|
|
|
// Create a new VideoToolbox decoder session if one already exists
|
|
if (mDecoderSession != nullptr) {
|
|
// TODO: Once we implement async frame readback, we'll need to flush all of the frames here and
|
|
// store them somewhere for the guest to read later.
|
|
VTDecompressionSessionInvalidate(mDecoderSession);
|
|
CFRelease(mDecoderSession);
|
|
mDecoderSession = nullptr;
|
|
if (mDecodedFrame) {
|
|
CVPixelBufferRelease(mDecodedFrame);
|
|
mDecodedFrame = nullptr;
|
|
}
|
|
}
|
|
|
|
CMVideoCodecType codecType = kCMVideoCodecType_H264;
|
|
CFMutableDictionaryRef decoder_spec = CFDictionaryCreateMutable(kCFAllocatorDefault,
|
|
0,
|
|
&kCFTypeDictionaryKeyCallBacks,
|
|
&kCFTypeDictionaryValueCallBacks);
|
|
CFDictionarySetValue(decoder_spec,
|
|
kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
|
|
kCFBooleanTrue);
|
|
|
|
CFDictionaryRef bufAttr = createOutputBufferAttributes(mOutputWidth,
|
|
mOutputHeight,
|
|
toNativePixelFormat(mOutPixFmt));
|
|
|
|
VTDecompressionOutputCallbackRecord decoderCb;
|
|
decoderCb.decompressionOutputCallback = videoToolboxDecompressCallback;
|
|
decoderCb.decompressionOutputRefCon = this;
|
|
|
|
OSStatus status;
|
|
status = VTDecompressionSessionCreate(NULL, // allocator
|
|
mCmFmtDesc, // videoFormatDescription
|
|
decoder_spec, // videoDecoderSpecification
|
|
bufAttr, // destinationImageBufferAttributes
|
|
&decoderCb, // outputCallback
|
|
&mDecoderSession); // decompressionSessionOut
|
|
|
|
if (decoder_spec) {
|
|
CFRelease(decoder_spec);
|
|
}
|
|
if (bufAttr) {
|
|
CFRelease(bufAttr);
|
|
}
|
|
|
|
mIsInFlush = false;
|
|
mState = DecoderState::BAD_STATE;
|
|
switch (status) {
|
|
case kVTVideoDecoderNotAvailableNowErr:
|
|
H264_DPRINT("VideoToolbox session not available");
|
|
return;
|
|
case kVTVideoDecoderUnsupportedDataFormatErr:
|
|
H264_DPRINT("VideoToolbox does not support this format");
|
|
return;
|
|
case kVTVideoDecoderMalfunctionErr:
|
|
H264_DPRINT("VideoToolbox malfunction");
|
|
return;
|
|
case kVTVideoDecoderBadDataErr:
|
|
H264_DPRINT("VideoToolbox reported invalid data");
|
|
return;
|
|
case 0:
|
|
H264_DPRINT("VideoToolbox session created");
|
|
mState = DecoderState::GOOD_STATE;
|
|
return;
|
|
default:
|
|
H264_DPRINT("Unknown VideoToolbox session creation error %d", status);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void MediaH264DecoderVideoToolBox::save(base::Stream* stream) const {
|
|
stream->putBe32(mParser.version());
|
|
stream->putBe32(mWidth);
|
|
stream->putBe32(mHeight);
|
|
stream->putBe32((int)mOutPixFmt);
|
|
|
|
const int hasContext = mDecoderSession != nullptr ? 1 : 0;
|
|
stream->putBe32(hasContext);
|
|
|
|
if (mImageReady) {
|
|
mSnapshotState.saveDecodedFrame(
|
|
mSavedDecodedFrame, mOutputWidth, mOutputHeight,
|
|
ColorAspects{}, mOutputPts);
|
|
} else {
|
|
mSnapshotState.savedDecodedFrame.data.clear();
|
|
}
|
|
H264_DPRINT("saving packets now %d",
|
|
(int)(mSnapshotState.savedPackets.size()));
|
|
mSnapshotState.save(stream);
|
|
}
|
|
|
|
bool MediaH264DecoderVideoToolBox::load(base::Stream* stream) {
|
|
mIsLoadingFromSnapshot = true;
|
|
uint32_t version = stream->getBe32();
|
|
mParser = H264PingInfoParser{version};
|
|
|
|
mWidth = stream->getBe32();
|
|
mHeight = stream->getBe32();
|
|
mOutPixFmt = (PixelFormat)stream->getBe32();
|
|
|
|
const int hasContext = stream->getBe32();
|
|
if (hasContext) {
|
|
initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
|
|
}
|
|
mSnapshotState.load(stream);
|
|
if (hasContext && mSnapshotState.sps.size() > 0) {
|
|
oneShotDecode(mSnapshotState.sps, 0);
|
|
if (mSnapshotState.pps.size() > 0) {
|
|
oneShotDecode(mSnapshotState.pps, 0);
|
|
if (mSnapshotState.savedPackets.size() > 0) {
|
|
for (int i = 0; i < mSnapshotState.savedPackets.size(); ++i) {
|
|
PacketInfo& pkt = mSnapshotState.savedPackets[i];
|
|
oneShotDecode(pkt.data, pkt.pts);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (mSnapshotState.savedDecodedFrame.data.size() > 0) {
|
|
mSavedDecodedFrame = std::move(mSnapshotState.savedDecodedFrame.data);
|
|
mOutBufferSize = mSnapshotState.savedDecodedFrame.data.size();
|
|
mOutputWidth = mSnapshotState.savedDecodedFrame.width;
|
|
mOutputHeight = mSnapshotState.savedDecodedFrame.height;
|
|
mOutputPts = mSnapshotState.savedDecodedFrame.pts;
|
|
mImageReady = true;
|
|
} else {
|
|
mOutputWidth = mWidth;
|
|
mOutputHeight = mHeight;
|
|
mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
|
|
mImageReady = false;
|
|
}
|
|
mIsLoadingFromSnapshot = false;
|
|
return true;
|
|
}
|
|
|
|
} // namespace emulation
|
|
} // namespace android
|