You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
432 lines
18 KiB
432 lines
18 KiB
4 months ago
|
// Copyright 2018 The Gemmlowp Authors. All Rights Reserved.
|
||
|
|
//
|
||
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
// you may not use this file except in compliance with the License.
|
||
|
|
// You may obtain a copy of the License at
|
||
|
|
//
|
||
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
//
|
||
|
|
// Unless required by applicable law or agreed to in writing, software
|
||
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
// See the License for the specific language governing permissions and
|
||
|
|
// limitations under the License.
|
||
|
|
|
||
|
|
// pack_msa.h: optimized MSA specializations of the templates in pack.h.
|
||
|
|
|
||
|
|
#ifndef GEMMLOWP_INTERNAL_PACK_MSA_H_
|
||
|
|
#define GEMMLOWP_INTERNAL_PACK_MSA_H_
|
||
|
|
|
||
|
|
#include "pack.h"
|
||
|
|
|
||
|
|
#include <msa.h>
|
||
|
|
|
||
|
|
namespace gemmlowp {
|
||
|
|
|
||
|
|
typedef SideMap<const std::uint8_t, SideMapOrder::WidthMajor>
|
||
|
|
WidthMajorUint8SideMap;
|
||
|
|
|
||
|
|
template <int Cells>
|
||
|
|
using DepthMajorSideFormatNCells4x2 = KernelSideFormat<CellFormat<4, 2>, Cells>;
|
||
|
|
|
||
|
|
template <int Cells>
|
||
|
|
class PackingRegisterBlock<
|
||
|
|
WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<DepthMajorSideFormatNCells4x2<Cells>>>
|
||
|
|
: public PackingRegisterBlockBase<
|
||
|
|
WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<DepthMajorSideFormatNCells4x2<Cells>>> {
|
||
|
|
public:
|
||
|
|
typedef DepthMajorSideFormatNCells4x2<Cells> KernelSideFormat;
|
||
|
|
typedef typename KernelSideFormat::Cell CellFormat;
|
||
|
|
static constexpr int kCells = KernelSideFormat::kCells;
|
||
|
|
static const int kCellWidth = CellFormat::kWidth;
|
||
|
|
static const int kKernelWidth = CellFormat::kWidth * kCells;
|
||
|
|
static const int kCellDepth = CellFormat::kDepth;
|
||
|
|
static const int kCellSize = CellFormat::kSize;
|
||
|
|
|
||
|
|
void Pack(PackedSideBlock<KernelSideFormat>* dst, int start_width) {
|
||
|
|
std::uint8_t* dst_ptr = dst->current_data();
|
||
|
|
const std::uint8_t* const src_ptr = this->complete_src_.data();
|
||
|
|
const int stride = this->complete_src_.stride();
|
||
|
|
// Load source WidthMajor data
|
||
|
|
v16i8 src_lines[4 * kCells];
|
||
|
|
for (int i = 0; i < 4 * kCells; i++) {
|
||
|
|
src_lines[i] = __builtin_msa_ld_b(
|
||
|
|
const_cast<std::uint8_t*>(src_ptr + i * stride), 0);
|
||
|
|
}
|
||
|
|
// Reorder the data within registers to make DepthMajor 4x2 cells
|
||
|
|
v16i8 src_lines_intertwined_2x[2 * kCells][2];
|
||
|
|
for (int i = 0; i < kCells; i++) {
|
||
|
|
src_lines_intertwined_2x[2 * i][0] =
|
||
|
|
__builtin_msa_ilvr_b(src_lines[4 * i + 2], src_lines[4 * i]);
|
||
|
|
src_lines_intertwined_2x[2 * i][1] =
|
||
|
|
__builtin_msa_ilvl_b(src_lines[4 * i + 2], src_lines[4 * i]);
|
||
|
|
src_lines_intertwined_2x[2 * i + 1][0] =
|
||
|
|
__builtin_msa_ilvr_b(src_lines[4 * i + 3], src_lines[4 * i + 1]);
|
||
|
|
src_lines_intertwined_2x[2 * i + 1][1] =
|
||
|
|
__builtin_msa_ilvl_b(src_lines[4 * i + 3], src_lines[4 * i + 1]);
|
||
|
|
}
|
||
|
|
v16i8 src_lines_intertwined_4x[2 * kCells][2];
|
||
|
|
for (int i = 0; i < kCells; i++) {
|
||
|
|
src_lines_intertwined_4x[2 * i][0] =
|
||
|
|
__builtin_msa_ilvr_b(src_lines_intertwined_2x[2 * i + 1][0],
|
||
|
|
src_lines_intertwined_2x[2 * i][0]);
|
||
|
|
src_lines_intertwined_4x[2 * i][1] =
|
||
|
|
__builtin_msa_ilvl_b(src_lines_intertwined_2x[2 * i + 1][0],
|
||
|
|
src_lines_intertwined_2x[2 * i][0]);
|
||
|
|
src_lines_intertwined_4x[2 * i + 1][0] =
|
||
|
|
__builtin_msa_ilvr_b(src_lines_intertwined_2x[2 * i + 1][1],
|
||
|
|
src_lines_intertwined_2x[2 * i][1]);
|
||
|
|
src_lines_intertwined_4x[2 * i + 1][1] =
|
||
|
|
__builtin_msa_ilvl_b(src_lines_intertwined_2x[2 * i + 1][1],
|
||
|
|
src_lines_intertwined_2x[2 * i][1]);
|
||
|
|
}
|
||
|
|
// Store the resulting DepthMajor 4x2 cells in the destination packed block
|
||
|
|
for (int outer = 0; outer < 2; outer++) {
|
||
|
|
for (int inner = 0; inner < 2; inner++) {
|
||
|
|
if (kCells % 2 == 0) {
|
||
|
|
for (int cell = 0; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvr_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
for (int cell = 0; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvl_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
// Store even number of low vector halves.
|
||
|
|
for (int cell = 0; cell < kCells - 1; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvr_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
// Store last low half and first high half.
|
||
|
|
v2i64 tmp = reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * 0 + outer][inner]);
|
||
|
|
tmp = __builtin_msa_insve_d(
|
||
|
|
tmp, 0,
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (kCells - 1) + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
// Store even number of high vector halves.
|
||
|
|
for (int cell = 1; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvl_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Compute sums across the depth dimension
|
||
|
|
v8i16 sums_of_2_cells[kCells][4];
|
||
|
|
const v16i8 zeroes = __builtin_msa_ldi_b(0);
|
||
|
|
for (int outer = 0; outer < 2; outer++) {
|
||
|
|
for (int inner = 0; inner < 2; inner++) {
|
||
|
|
int i = 2 * outer + inner;
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
v8i16 tmp0 = reinterpret_cast<v8i16>(__builtin_msa_ilvr_b(
|
||
|
|
zeroes, src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
v8i16 tmp1 = reinterpret_cast<v8i16>(__builtin_msa_ilvl_b(
|
||
|
|
zeroes, src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
sums_of_2_cells[cell][i] = __builtin_msa_addv_h(tmp0, tmp1);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
v4i32 sums_of_4_cells[kCells][4];
|
||
|
|
for (int i = 0; i < 4; i++) {
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
v4i32 tmp0 = reinterpret_cast<v4i32>(__builtin_msa_ilvr_h(
|
||
|
|
reinterpret_cast<v8i16>(zeroes), sums_of_2_cells[cell][i]));
|
||
|
|
v4i32 tmp1 = reinterpret_cast<v4i32>(__builtin_msa_ilvl_h(
|
||
|
|
reinterpret_cast<v8i16>(zeroes), sums_of_2_cells[cell][i]));
|
||
|
|
sums_of_4_cells[cell][i] = __builtin_msa_addv_w(tmp0, tmp1);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Update the sums_of_each_slice vector
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
v4i32 s01 = __builtin_msa_addv_w(sums_of_4_cells[cell][0],
|
||
|
|
sums_of_4_cells[cell][1]);
|
||
|
|
v4i32 s23 = __builtin_msa_addv_w(sums_of_4_cells[cell][2],
|
||
|
|
sums_of_4_cells[cell][3]);
|
||
|
|
v4i32 s = __builtin_msa_addv_w(s01, s23);
|
||
|
|
std::int32_t* sums_of_each_slice_ptr =
|
||
|
|
dst->sums_of_each_slice() + start_width + 4 * cell;
|
||
|
|
v4i32 tmp = __builtin_msa_ld_w(sums_of_each_slice_ptr, 0);
|
||
|
|
tmp = __builtin_msa_addv_w(tmp, s);
|
||
|
|
__builtin_msa_st_w(tmp, sums_of_each_slice_ptr, 0);
|
||
|
|
}
|
||
|
|
dst->seek_forward_n_cells(kCells * kRegisterSize / kCellDepth);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
template <int Cells>
|
||
|
|
using WidthMajorSideFormatNCells4x2 =
|
||
|
|
KernelSideFormat<CellFormat<4, 2, CellOrder::WidthMajor>, Cells>;
|
||
|
|
|
||
|
|
template <int Cells>
|
||
|
|
class PackingRegisterBlock<
|
||
|
|
WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<WidthMajorSideFormatNCells4x2<Cells>>>
|
||
|
|
: public PackingRegisterBlockBase<
|
||
|
|
WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<WidthMajorSideFormatNCells4x2<Cells>>> {
|
||
|
|
public:
|
||
|
|
typedef WidthMajorSideFormatNCells4x2<Cells> KernelSideFormat;
|
||
|
|
typedef typename KernelSideFormat::Cell CellFormat;
|
||
|
|
static constexpr int kCells = KernelSideFormat::kCells;
|
||
|
|
static const int kCellWidth = CellFormat::kWidth;
|
||
|
|
static const int kKernelWidth = CellFormat::kWidth * kCells;
|
||
|
|
static const int kCellDepth = CellFormat::kDepth;
|
||
|
|
static const int kCellSize = CellFormat::kSize;
|
||
|
|
|
||
|
|
void Pack(PackedSideBlock<KernelSideFormat>* dst, int start_width) {
|
||
|
|
std::uint8_t* dst_ptr = dst->current_data();
|
||
|
|
const std::uint8_t* src_ptr = this->complete_src_.data();
|
||
|
|
const int stride = this->complete_src_.stride();
|
||
|
|
// Load source WidthMajor data
|
||
|
|
v8i16 src_lines[kCells * 4];
|
||
|
|
for (int i = 0; i < kCells; i++) {
|
||
|
|
#define GEMMLOWP_UNROLLED_LOOP_ITER(k) \
|
||
|
|
src_lines[4 * i + k] = \
|
||
|
|
__builtin_msa_ld_h(const_cast<std::uint8_t*>(src_ptr), 0); \
|
||
|
|
src_ptr += stride;
|
||
|
|
|
||
|
|
GEMMLOWP_UNROLLED_LOOP_ITER(0)
|
||
|
|
GEMMLOWP_UNROLLED_LOOP_ITER(1)
|
||
|
|
GEMMLOWP_UNROLLED_LOOP_ITER(2)
|
||
|
|
GEMMLOWP_UNROLLED_LOOP_ITER(3)
|
||
|
|
|
||
|
|
#undef GEMMLOWP_UNROLLED_LOOP_ITER
|
||
|
|
}
|
||
|
|
// Reorder the data within registers to make WidthMajor 4x2 cells
|
||
|
|
v8i16 src_lines_intertwined_2x[2 * kCells][2];
|
||
|
|
for (int i = 0; i < kCells; i++) {
|
||
|
|
src_lines_intertwined_2x[2 * i][0] =
|
||
|
|
__builtin_msa_ilvr_h(src_lines[4 * i + 2], src_lines[4 * i]);
|
||
|
|
src_lines_intertwined_2x[2 * i][1] =
|
||
|
|
__builtin_msa_ilvl_h(src_lines[4 * i + 2], src_lines[4 * i]);
|
||
|
|
src_lines_intertwined_2x[2 * i + 1][0] =
|
||
|
|
__builtin_msa_ilvr_h(src_lines[4 * i + 3], src_lines[4 * i + 1]);
|
||
|
|
src_lines_intertwined_2x[2 * i + 1][1] =
|
||
|
|
__builtin_msa_ilvl_h(src_lines[4 * i + 3], src_lines[4 * i + 1]);
|
||
|
|
}
|
||
|
|
v8i16 src_lines_intertwined_4x[2 * kCells][2];
|
||
|
|
for (int i = 0; i < kCells; i++) {
|
||
|
|
src_lines_intertwined_4x[2 * i][0] =
|
||
|
|
__builtin_msa_ilvr_h(src_lines_intertwined_2x[2 * i + 1][0],
|
||
|
|
src_lines_intertwined_2x[2 * i][0]);
|
||
|
|
src_lines_intertwined_4x[2 * i][1] =
|
||
|
|
__builtin_msa_ilvl_h(src_lines_intertwined_2x[2 * i + 1][0],
|
||
|
|
src_lines_intertwined_2x[2 * i][0]);
|
||
|
|
src_lines_intertwined_4x[2 * i + 1][0] =
|
||
|
|
__builtin_msa_ilvr_h(src_lines_intertwined_2x[2 * i + 1][1],
|
||
|
|
src_lines_intertwined_2x[2 * i][1]);
|
||
|
|
src_lines_intertwined_4x[2 * i + 1][1] =
|
||
|
|
__builtin_msa_ilvl_h(src_lines_intertwined_2x[2 * i + 1][1],
|
||
|
|
src_lines_intertwined_2x[2 * i][1]);
|
||
|
|
}
|
||
|
|
// Store the resulting WidthMajor 4x2 cells in the destination packed block
|
||
|
|
for (int outer = 0; outer < 2; outer++) {
|
||
|
|
for (int inner = 0; inner < 2; inner++) {
|
||
|
|
if (kCells % 2 == 0) {
|
||
|
|
for (int cell = 0; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvr_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
for (int cell = 0; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvl_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
// Store even number of low vector halves.
|
||
|
|
for (int cell = 0; cell < kCells - 1; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvr_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
// Store last low half and first high half.
|
||
|
|
v2i64 tmp = reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * 0 + outer][inner]);
|
||
|
|
tmp = __builtin_msa_insve_d(
|
||
|
|
tmp, 0,
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (kCells - 1) + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
// Store even number of high vector halves.
|
||
|
|
for (int cell = 1; cell < kCells; cell += 2) {
|
||
|
|
v2i64 tmp = __builtin_msa_ilvl_d(
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * (cell + 1) + outer][inner]),
|
||
|
|
reinterpret_cast<v2i64>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]));
|
||
|
|
__builtin_msa_st_b(reinterpret_cast<v16i8>(tmp), dst_ptr, 0);
|
||
|
|
dst_ptr += 16;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Compute sums across the depth dimension
|
||
|
|
v8i16 sums_of_2[kCells][4];
|
||
|
|
for (int outer = 0; outer < 2; outer++) {
|
||
|
|
for (int inner = 0; inner < 2; inner++) {
|
||
|
|
int i = 2 * outer + inner;
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
sums_of_2[cell][i] = reinterpret_cast<v8i16>(__builtin_msa_hadd_u_h(
|
||
|
|
reinterpret_cast<v16u8>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner]),
|
||
|
|
reinterpret_cast<v16u8>(
|
||
|
|
src_lines_intertwined_4x[2 * cell + outer][inner])));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
v8i16 sums_of_4[kCells][2];
|
||
|
|
for (int i = 0; i < 2; i++) {
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
sums_of_4[cell][i] = __builtin_msa_addv_h(sums_of_2[cell][2 * i],
|
||
|
|
sums_of_2[cell][2 * i + 1]);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
v8i16 sums_of_8[kCells];
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
sums_of_8[cell] =
|
||
|
|
__builtin_msa_addv_h(sums_of_4[cell][0], sums_of_4[cell][1]);
|
||
|
|
}
|
||
|
|
|
||
|
|
v4i32 sums_of_16[kCells];
|
||
|
|
const v8i16 zeroes = __builtin_msa_ldi_h(0);
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
sums_of_16[cell] = reinterpret_cast<v4i32>(
|
||
|
|
__builtin_msa_ilvr_h(zeroes, sums_of_8[cell]));
|
||
|
|
v8i16 tmp = __builtin_msa_ilvl_h(zeroes, sums_of_8[cell]);
|
||
|
|
sums_of_16[cell] =
|
||
|
|
__builtin_msa_addv_w(sums_of_16[cell], reinterpret_cast<v4i32>(tmp));
|
||
|
|
}
|
||
|
|
// Update the sums_of_each_slice vector
|
||
|
|
for (int cell = 0; cell < kCells; cell++) {
|
||
|
|
std::int32_t* sums_of_each_slice_ptr =
|
||
|
|
dst->sums_of_each_slice() + start_width + 4 * cell;
|
||
|
|
v4i32 tmp = __builtin_msa_ld_w(sums_of_each_slice_ptr, 0);
|
||
|
|
tmp = __builtin_msa_addv_w(tmp, sums_of_16[cell]);
|
||
|
|
__builtin_msa_st_w(tmp, sums_of_each_slice_ptr, 0);
|
||
|
|
}
|
||
|
|
dst->seek_forward_n_cells(kCells * kRegisterSize / kCellDepth);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
template <int Width>
|
||
|
|
using Int8FastKernelFormat =
|
||
|
|
KernelSideFormatInt8<CellFormat<Width, 16, CellOrder::WidthMajor>, 1>;
|
||
|
|
|
||
|
|
template <int Width>
|
||
|
|
class PackingRegisterBlock<WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<Int8FastKernelFormat<Width>>>
|
||
|
|
: public PackingRegisterBlockBase<
|
||
|
|
WidthMajorUint8SideMap,
|
||
|
|
PackedSideBlock<Int8FastKernelFormat<Width>>> {
|
||
|
|
public:
|
||
|
|
static_assert(Width == 2 || Width == 4, "");
|
||
|
|
typedef Int8FastKernelFormat<Width> KernelSideFormat;
|
||
|
|
typedef typename KernelSideFormat::Cell CellFormat;
|
||
|
|
static const int kCells = KernelSideFormat::kCells;
|
||
|
|
static const int kCellWidth = CellFormat::kWidth;
|
||
|
|
static const int kKernelWidth = CellFormat::kWidth * kCells;
|
||
|
|
static const int kCellDepth = CellFormat::kDepth;
|
||
|
|
static const int kCellSize = CellFormat::kSize;
|
||
|
|
|
||
|
|
void Pack(PackedSideBlock<KernelSideFormat>* dst, int start_width) {
|
||
|
|
std::int32_t* sums_ptr = dst->sums_of_each_slice() + start_width;
|
||
|
|
std::uint8_t* dst_ptr = dst->current_data();
|
||
|
|
const std::uint8_t* const src_ptr = this->complete_src_.data();
|
||
|
|
const int stride = this->complete_src_.stride();
|
||
|
|
// Load source WidthMajor data.
|
||
|
|
v16i8 src_lines[Width];
|
||
|
|
for (int i = 0; i < Width; i++) {
|
||
|
|
src_lines[i] = __builtin_msa_ld_b(
|
||
|
|
const_cast<std::uint8_t*>(src_ptr + i * stride), 0);
|
||
|
|
}
|
||
|
|
for (int i = 0; i < Width; i++) {
|
||
|
|
// Subtract 128 by inverting bit 7.
|
||
|
|
src_lines[i] = reinterpret_cast<v16i8>(
|
||
|
|
__builtin_msa_bnegi_b(reinterpret_cast<v16u8>(src_lines[i]), 7));
|
||
|
|
}
|
||
|
|
for (int i = 0; i < Width; i++) {
|
||
|
|
__builtin_msa_st_b(src_lines[i], dst_ptr + 16 * i, 0);
|
||
|
|
}
|
||
|
|
v8i16 sums2[Width];
|
||
|
|
for (int i = 0; i < Width; i++) {
|
||
|
|
sums2[i] = __builtin_msa_hadd_s_h(src_lines[i], src_lines[i]);
|
||
|
|
}
|
||
|
|
v4i32 sums4_wide[Width];
|
||
|
|
for (int i = 0; i < Width; i++) {
|
||
|
|
sums4_wide[i] = __builtin_msa_hadd_s_w(sums2[i], sums2[i]);
|
||
|
|
}
|
||
|
|
v8i16 sums4[Width / 2];
|
||
|
|
for (int i = 0; i < Width / 2; i++) {
|
||
|
|
sums4[i] = __builtin_msa_pckev_h(
|
||
|
|
reinterpret_cast<v8i16>(sums4_wide[2 * i + 1]),
|
||
|
|
reinterpret_cast<v8i16>(sums4_wide[2 * i]));
|
||
|
|
}
|
||
|
|
v4i32 sums8_wide[Width / 2];
|
||
|
|
for (int i = 0; i < Width / 2; i++) {
|
||
|
|
sums8_wide[i] = __builtin_msa_hadd_s_w(sums4[i], sums4[i]);
|
||
|
|
}
|
||
|
|
if (Width == 4) {
|
||
|
|
v4i32 sum = __builtin_msa_ld_w(const_cast<std::int32_t*>(sums_ptr), 0);
|
||
|
|
v8i16 sums8 = __builtin_msa_pckev_h(
|
||
|
|
reinterpret_cast<v8i16>(sums8_wide[1]),
|
||
|
|
reinterpret_cast<v8i16>(sums8_wide[0]));
|
||
|
|
v4i32 sums16 = __builtin_msa_hadd_s_w(sums8, sums8);
|
||
|
|
sum = __builtin_msa_addv_w(sum, sums16);
|
||
|
|
__builtin_msa_st_w(sum, sums_ptr, 0);
|
||
|
|
} else {
|
||
|
|
assert(Width == 2);
|
||
|
|
std::int32_t sum[2] = { sums_ptr[0], sums_ptr[1] };
|
||
|
|
v2i64 sums16 = __builtin_msa_hadd_s_d(sums8_wide[0], sums8_wide[0]);
|
||
|
|
sum[0] += __builtin_msa_copy_s_w(reinterpret_cast<v4i32>(sums16), 0);
|
||
|
|
sum[1] += __builtin_msa_copy_s_w(reinterpret_cast<v4i32>(sums16), 2);
|
||
|
|
sums_ptr[0] = sum[0];
|
||
|
|
sums_ptr[1] = sum[1];
|
||
|
|
}
|
||
|
|
dst->seek_forward_n_cells(1);
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace gemmlowp
|
||
|
|
|
||
|
|
#endif // GEMMLOWP_INTERNAL_PACK_MSA_H_
|