You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
328 lines
17 KiB
328 lines
17 KiB
/*
|
|
* Copyright (C) 2018 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
#define LOG_TAG "Operations"
|
|
|
|
#include <tensorflow/lite/kernels/internal/common.h>
|
|
|
|
#include <algorithm>
|
|
#include <cfloat>
|
|
#include <cmath>
|
|
#include <vector>
|
|
|
|
#include "CpuOperationUtils.h"
|
|
#include "Operations.h"
|
|
#include "Tracing.h"
|
|
|
|
namespace android {
|
|
namespace nn {
|
|
|
|
#define ANDROID_NN_GROUPED_CONV_PARAMETERS \
|
|
uint32_t numBatches = getSizeOfDimension(inputShape, 0); \
|
|
uint32_t inputHeight = getSizeOfDimension(inputShape, 1); \
|
|
uint32_t inputWidth = getSizeOfDimension(inputShape, 2); \
|
|
uint32_t inputDepth = getSizeOfDimension(inputShape, 3); \
|
|
uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \
|
|
uint32_t filterWidth = getSizeOfDimension(filterShape, 2); \
|
|
uint32_t filterDepth = getSizeOfDimension(filterShape, 3); \
|
|
uint32_t outputHeight = getSizeOfDimension(outputShape, 1); \
|
|
uint32_t outputWidth = getSizeOfDimension(outputShape, 2); \
|
|
uint32_t outputDepth = getSizeOfDimension(outputShape, 3); \
|
|
uint32_t outputGroupDepth = outputDepth / numGroups;
|
|
|
|
bool groupedConvFloat32(const float* inputData, const Shape& inputShape, const float* filterData,
|
|
const Shape& filterShape, const float* biasData, const Shape& biasShape,
|
|
int32_t padding_left, int32_t padding_right, int32_t padding_top,
|
|
int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
|
|
int32_t numGroups, int32_t activation, float* outputData,
|
|
const Shape& outputShape) {
|
|
NNTRACE_TRANS("groupConvFloat32");
|
|
ANDROID_NN_GROUPED_CONV_PARAMETERS
|
|
|
|
float output_activation_min = 0.0f, output_activation_max = 0.0f;
|
|
CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max);
|
|
|
|
const float* inputBase = inputData;
|
|
float* outPtr = outputData;
|
|
for (uint32_t b = 0; b < numBatches; b++) {
|
|
for (uint32_t h = 0; h < outputHeight; h++) {
|
|
for (uint32_t w = 0; w < outputWidth; w++) {
|
|
const float* filterBase = filterData;
|
|
for (uint32_t g = 0; g < numGroups; g++) {
|
|
for (uint32_t d = 0; d < outputGroupDepth; d++) {
|
|
int32_t wInputOrigin =
|
|
static_cast<int32_t>(w) * stride_width - padding_left;
|
|
int32_t hInputOrigin =
|
|
static_cast<int32_t>(h) * stride_height - padding_top;
|
|
float sum = 0.0f;
|
|
for (uint32_t i = 0; i < filterHeight; i++) {
|
|
for (uint32_t j = 0; j < filterWidth; j++) {
|
|
for (uint32_t k = 0; k < filterDepth; k++) {
|
|
int32_t hInput = hInputOrigin + static_cast<int32_t>(i);
|
|
int32_t wInput = wInputOrigin + static_cast<int32_t>(j);
|
|
uint32_t dInput = filterDepth * g + k;
|
|
if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
|
|
wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
|
|
uint32_t filterIndex =
|
|
i * filterWidth * filterDepth + j * filterDepth + k;
|
|
uint32_t inputIndex = hInput * inputWidth * inputDepth +
|
|
wInput * inputDepth + dInput;
|
|
sum += filterBase[filterIndex] * inputBase[inputIndex];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sum += biasData[g * outputGroupDepth + d];
|
|
sum = std::max(std::min(sum, output_activation_max), output_activation_min);
|
|
outPtr[d] = sum;
|
|
filterBase += filterHeight * filterWidth * filterDepth;
|
|
}
|
|
outPtr += outputGroupDepth;
|
|
}
|
|
}
|
|
}
|
|
inputBase += inputHeight * inputWidth * inputDepth;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
template <typename T>
|
|
bool groupedConvQuant8(const T* inputData, const Shape& inputShape, const T* filterData,
|
|
const Shape& filterShape, const int32_t* biasData, const Shape& biasShape,
|
|
int32_t padding_left, int32_t padding_right, int32_t padding_top,
|
|
int32_t padding_bottom, int32_t stride_width, int32_t stride_height,
|
|
int32_t numGroups, int32_t activation, T* outputData,
|
|
const Shape& outputShape) {
|
|
NNTRACE_TRANS("groupConvQuant8");
|
|
ANDROID_NN_GROUPED_CONV_PARAMETERS
|
|
|
|
int32_t inputOffset = -inputShape.offset;
|
|
int32_t filterOffset = -filterShape.offset;
|
|
int32_t outputOffset = outputShape.offset;
|
|
|
|
double realMultiplier = 0.0;
|
|
int32_t outputMultiplier = 0;
|
|
int32_t outputShift = 0;
|
|
NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, filterShape, biasShape, outputShape,
|
|
&realMultiplier));
|
|
int exponent;
|
|
NN_RET_CHECK(QuantizeMultiplier(realMultiplier, &outputMultiplier, &exponent));
|
|
outputShift = -exponent;
|
|
|
|
int32_t output_activation_min = 0, output_activation_max = 0;
|
|
CalculateActivationRange<T>(activation, outputShape, &output_activation_min,
|
|
&output_activation_max);
|
|
|
|
const T* inputBase = inputData;
|
|
T* outPtr = outputData;
|
|
for (uint32_t b = 0; b < numBatches; b++) {
|
|
for (uint32_t h = 0; h < outputHeight; h++) {
|
|
for (uint32_t w = 0; w < outputWidth; w++) {
|
|
const T* filterBase = filterData;
|
|
for (uint32_t g = 0; g < numGroups; g++) {
|
|
for (uint32_t d = 0; d < outputGroupDepth; d++) {
|
|
int32_t wInputOrigin =
|
|
static_cast<int32_t>(w) * stride_width - padding_left;
|
|
int32_t hInputOrigin =
|
|
static_cast<int32_t>(h) * stride_height - padding_top;
|
|
int32_t sum = 0.0f;
|
|
for (uint32_t i = 0; i < filterHeight; i++) {
|
|
for (uint32_t j = 0; j < filterWidth; j++) {
|
|
for (uint32_t k = 0; k < filterDepth; k++) {
|
|
int32_t hInput = hInputOrigin + static_cast<int32_t>(i);
|
|
int32_t wInput = wInputOrigin + static_cast<int32_t>(j);
|
|
uint32_t dInput = filterDepth * g + k;
|
|
if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
|
|
wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
|
|
uint32_t filterIndex =
|
|
i * filterWidth * filterDepth + j * filterDepth + k;
|
|
uint32_t inputIndex = hInput * inputWidth * inputDepth +
|
|
wInput * inputDepth + dInput;
|
|
sum += (static_cast<int32_t>(filterBase[filterIndex]) +
|
|
filterOffset) *
|
|
(static_cast<int32_t>(inputBase[inputIndex]) +
|
|
inputOffset);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
sum += biasData[g * outputGroupDepth + d];
|
|
sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier,
|
|
-outputShift);
|
|
sum += outputOffset;
|
|
sum = std::max(std::min(sum, output_activation_max), output_activation_min);
|
|
outPtr[d] = static_cast<T>(sum);
|
|
filterBase += filterHeight * filterWidth * filterDepth;
|
|
}
|
|
outPtr += outputGroupDepth;
|
|
}
|
|
}
|
|
}
|
|
inputBase += inputHeight * inputWidth * inputDepth;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
template bool groupedConvQuant8<int8_t>(const int8_t* inputData, const Shape& inputShape,
|
|
const int8_t* filterData, const Shape& filterShape,
|
|
const int32_t* biasData, const Shape& biasShape,
|
|
int32_t padding_left, int32_t padding_right,
|
|
int32_t padding_top, int32_t padding_bottom,
|
|
int32_t stride_width, int32_t stride_height,
|
|
int32_t numGroups, int32_t activation, int8_t* outputData,
|
|
const Shape& outputShape);
|
|
|
|
template bool groupedConvQuant8<uint8_t>(const uint8_t* inputData, const Shape& inputShape,
|
|
const uint8_t* filterData, const Shape& filterShape,
|
|
const int32_t* biasData, const Shape& biasShape,
|
|
int32_t padding_left, int32_t padding_right,
|
|
int32_t padding_top, int32_t padding_bottom,
|
|
int32_t stride_width, int32_t stride_height,
|
|
int32_t numGroups, int32_t activation, uint8_t* outputData,
|
|
const Shape& outputShape);
|
|
|
|
template <typename T>
|
|
bool groupedConvQuant8PerChannel(const T* inputData, const Shape& inputShape,
|
|
const int8_t* filterData, const Shape& filterShape,
|
|
const float* filterScales, const int32_t* biasData,
|
|
const Shape& biasShape, int32_t padding_left,
|
|
int32_t padding_right, int32_t padding_top, int32_t padding_bottom,
|
|
int32_t stride_width, int32_t stride_height, int32_t numGroups,
|
|
int32_t activation, T* outputData, const Shape& outputShape) {
|
|
NNTRACE_TRANS("groupConvQuant8");
|
|
ANDROID_NN_GROUPED_CONV_PARAMETERS
|
|
|
|
int32_t inputOffset = -inputShape.offset;
|
|
int32_t outputOffset = outputShape.offset;
|
|
|
|
auto realMultiplier = std::vector<double>(outputDepth, .0f);
|
|
auto outputMultiplier = std::vector<int32_t>(outputDepth, 0);
|
|
auto outputShift = std::vector<int32_t>(outputDepth, 0);
|
|
|
|
for (int i = 0; i < outputDepth; ++i) {
|
|
Shape filterChannelShape = filterShape;
|
|
filterChannelShape.scale = filterScales[i];
|
|
Shape biasChannelShape = biasShape;
|
|
biasChannelShape.scale = filterScales[i] * inputShape.scale;
|
|
|
|
NN_RET_CHECK(GetQuantizedConvolutionMultipler(
|
|
inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i]));
|
|
int exponent;
|
|
NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent));
|
|
outputShift[i] = -exponent;
|
|
}
|
|
|
|
int32_t output_activation_min = 0, output_activation_max = 0;
|
|
CalculateActivationRange<T>(activation, outputShape, &output_activation_min,
|
|
&output_activation_max);
|
|
|
|
const T* inputBase = inputData;
|
|
T* outPtr = outputData;
|
|
for (uint32_t b = 0; b < numBatches; b++) {
|
|
for (uint32_t h = 0; h < outputHeight; h++) {
|
|
for (uint32_t w = 0; w < outputWidth; w++) {
|
|
const int8_t* filterBase = filterData;
|
|
for (uint32_t g = 0; g < numGroups; g++) {
|
|
for (uint32_t d = 0; d < outputGroupDepth; d++) {
|
|
int32_t wInputOrigin =
|
|
static_cast<int32_t>(w) * stride_width - padding_left;
|
|
int32_t hInputOrigin =
|
|
static_cast<int32_t>(h) * stride_height - padding_top;
|
|
int32_t sum = 0.0f;
|
|
for (uint32_t i = 0; i < filterHeight; i++) {
|
|
for (uint32_t j = 0; j < filterWidth; j++) {
|
|
for (uint32_t k = 0; k < filterDepth; k++) {
|
|
int32_t hInput = hInputOrigin + static_cast<int32_t>(i);
|
|
int32_t wInput = wInputOrigin + static_cast<int32_t>(j);
|
|
uint32_t dInput = filterDepth * g + k;
|
|
if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) &&
|
|
wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) {
|
|
uint32_t filterIndex =
|
|
i * filterWidth * filterDepth + j * filterDepth + k;
|
|
uint32_t inputIndex = hInput * inputWidth * inputDepth +
|
|
wInput * inputDepth + dInput;
|
|
sum += (static_cast<int32_t>(filterBase[filterIndex])) *
|
|
(static_cast<int32_t>(inputBase[inputIndex]) +
|
|
inputOffset);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
int channelIndex = g * outputGroupDepth + d;
|
|
sum += biasData[channelIndex];
|
|
sum = tflite::MultiplyByQuantizedMultiplier(
|
|
sum, outputMultiplier[channelIndex], -outputShift[channelIndex]);
|
|
sum += outputOffset;
|
|
sum = std::max(std::min(sum, output_activation_max), output_activation_min);
|
|
outPtr[d] = static_cast<T>(sum);
|
|
filterBase += filterHeight * filterWidth * filterDepth;
|
|
}
|
|
outPtr += outputGroupDepth;
|
|
}
|
|
}
|
|
}
|
|
inputBase += inputHeight * inputWidth * inputDepth;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool groupedConvFloat16(const _Float16* inputData, const Shape& inputShape,
|
|
const _Float16* filterData, const Shape& filterShape,
|
|
const _Float16* biasData, const Shape& biasShape, int32_t padding_left,
|
|
int32_t padding_right, int32_t padding_top, int32_t padding_bottom,
|
|
int32_t stride_width, int32_t stride_height, int32_t numGroups,
|
|
int32_t activation, _Float16* outputData, const Shape& outputShape) {
|
|
NNTRACE_TRANS("groupConvFloat16");
|
|
|
|
std::vector<float> inputData_float32(getNumberOfElements(inputShape));
|
|
std::vector<float> filterData_float32(getNumberOfElements(filterShape));
|
|
std::vector<float> biasData_float32(getNumberOfElements(biasShape));
|
|
std::vector<float> outputData_float32(getNumberOfElements(outputShape));
|
|
|
|
convertFloat16ToFloat32(inputData, &inputData_float32);
|
|
convertFloat16ToFloat32(filterData, &filterData_float32);
|
|
convertFloat16ToFloat32(biasData, &biasData_float32);
|
|
|
|
groupedConvFloat32(inputData_float32.data(), inputShape, filterData_float32.data(), filterShape,
|
|
biasData_float32.data(), biasShape, padding_left, padding_right, padding_top,
|
|
padding_bottom, stride_width, stride_height, numGroups, activation,
|
|
outputData_float32.data(), outputShape);
|
|
convertFloat32ToFloat16(outputData_float32, outputData);
|
|
|
|
return true;
|
|
}
|
|
|
|
template bool groupedConvQuant8PerChannel<uint8_t>(
|
|
const uint8_t* inputData, const Shape& inputShape, const int8_t* filterData,
|
|
const Shape& filterShape, const float* filterScales, const int32_t* biasData,
|
|
const Shape& biasShape, int32_t padding_left, int32_t padding_right, int32_t padding_top,
|
|
int32_t padding_bottom, int32_t stride_width, int32_t stride_height, int32_t numGroups,
|
|
int32_t activation, uint8_t* outputData, const Shape& outputShape);
|
|
|
|
template bool groupedConvQuant8PerChannel<int8_t>(
|
|
const int8_t* inputData, const Shape& inputShape, const int8_t* filterData,
|
|
const Shape& filterShape, const float* filterScales, const int32_t* biasData,
|
|
const Shape& biasShape, int32_t padding_left, int32_t padding_right, int32_t padding_top,
|
|
int32_t padding_bottom, int32_t stride_width, int32_t stride_height, int32_t numGroups,
|
|
int32_t activation, int8_t* outputData, const Shape& outputShape);
|
|
|
|
#undef ANDROID_NN_GROUPED_CONV_PARAMETERS
|
|
} // namespace nn
|
|
} // namespace android
|