You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1360 lines
54 KiB
1360 lines
54 KiB
/******************************************************************************
|
|
*
|
|
* Copyright (C) 2018 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at:
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*****************************************************************************
|
|
* Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
|
|
*/
|
|
|
|
/**
|
|
*******************************************************************************
|
|
* @file
|
|
* ihevc_resi_trans_neon.c
|
|
*
|
|
* @brief
|
|
* Contains definitions of functions for computing residue and fwd transform
|
|
*
|
|
* @author
|
|
* Ittiam
|
|
*
|
|
* @par List of Functions:
|
|
* - ihevc_resi_trans_4x4_neon()
|
|
* - ihevc_resi_trans_4x4_ttype1_neon()
|
|
* - ihevc_resi_trans_8x8_neon()
|
|
* - ihevc_resi_trans_16x16_neon()
|
|
* @remarks
|
|
* None
|
|
*
|
|
*******************************************************************************
|
|
*/
|
|
|
|
/*****************************************************************************/
|
|
/* File Includes */
|
|
/*****************************************************************************/
|
|
/* System include files */
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
|
|
/* System user files */
|
|
#include "ihevc_typedefs.h"
|
|
#include "ihevc_macros.h"
|
|
#include "ihevc_defs.h"
|
|
#include "ihevc_cmn_utils_neon.h"
|
|
|
|
#include "ihevc_trans_tables.h"
|
|
#include "ihevc_resi_trans.h"
|
|
|
|
/*****************************************************************************/
|
|
/* Function Definitions */
|
|
/*****************************************************************************/
|
|
UWORD32 ihevc_resi_trans_4x4_neon(
|
|
UWORD8 *pu1_src,
|
|
UWORD8 *pu1_pred,
|
|
WORD32 *pi4_temp,
|
|
WORD16 *pi2_dst,
|
|
WORD32 src_strd,
|
|
WORD32 pred_strd,
|
|
WORD32 dst_strd,
|
|
CHROMA_PLANE_ID_T e_chroma_plane)
|
|
{
|
|
UWORD32 sad;
|
|
uint8x16_t inp_buf, pred_buf;
|
|
int16x8_t diff_1, diff_2;
|
|
int16x4_t diff_1_low, diff_1_high, diff_2_low, diff_2_high;
|
|
int16x8_t e_01, o_32;
|
|
int16x4_t e_0, e_1, o_0, o_1;
|
|
int32x4_t e_0_a_e_1, e_0_s_e_1;
|
|
int32x4_t temp1, temp2, temp3, temp4;
|
|
int32x4_t o_1_m_trans_10, o_1_m_trans_11;
|
|
int32x4_t e_03, e_12, o_03, o_12;
|
|
int16x4_t out_0, out_1, out_2, out_3;
|
|
uint16x8_t abs;
|
|
uint32x4_t b;
|
|
uint64x2_t c;
|
|
|
|
(void)pi4_temp;
|
|
if(e_chroma_plane == NULL_PLANE)
|
|
{
|
|
inp_buf = load_unaligned_u8q(pu1_src, src_strd);
|
|
pred_buf = load_unaligned_u8q(pu1_pred, pred_strd);
|
|
}
|
|
else
|
|
{
|
|
inp_buf = load_unaligned_u8qi(pu1_src + e_chroma_plane, src_strd);
|
|
pred_buf = load_unaligned_u8qi(pu1_pred + e_chroma_plane, pred_strd);
|
|
}
|
|
|
|
abs = vabdl_u8(vget_low_u8(inp_buf), vget_low_u8(pred_buf));
|
|
abs = vabal_u8(abs, vget_high_u8(inp_buf), vget_high_u8(pred_buf));
|
|
b = vpaddlq_u16(abs);
|
|
c = vpaddlq_u32(b);
|
|
sad = vget_lane_u32(vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
|
|
vreinterpret_u32_u64(vget_high_u64(c))),
|
|
0);
|
|
|
|
diff_1 = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(inp_buf), vget_low_u8(pred_buf)));
|
|
diff_2 = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(inp_buf), vget_high_u8(pred_buf)));
|
|
|
|
diff_1_low = vget_low_s16(diff_1);
|
|
diff_1_high = vget_high_s16(diff_1);
|
|
diff_2_low = vget_low_s16(diff_2);
|
|
diff_2_high = vget_high_s16(diff_2);
|
|
|
|
transpose_s16_4x4d(&diff_1_low, &diff_1_high, &diff_2_low, &diff_2_high);
|
|
diff_1 = vcombine_s16(diff_1_low, diff_1_high);
|
|
diff_2 = vcombine_s16(diff_2_high, diff_2_low);
|
|
|
|
e_01 = vaddq_s16(diff_1, diff_2);
|
|
o_32 = vsubq_s16(diff_1, diff_2);
|
|
|
|
e_0 = vget_low_s16(e_01);
|
|
e_1 = vget_high_s16(e_01);
|
|
o_0 = vget_high_s16(o_32);
|
|
o_1 = vget_low_s16(o_32);
|
|
|
|
e_0_a_e_1 = vaddl_s16(e_0, e_1);
|
|
e_0_s_e_1 = vsubl_s16(e_0, e_1);
|
|
|
|
temp1 = vmulq_n_s32(e_0_a_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
|
|
temp2 = vmulq_n_s32(e_0_s_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
|
|
|
|
o_1_m_trans_10 = vmull_n_s16(o_1, (WORD32)g_ai2_ihevc_trans_4[1][0]);
|
|
o_1_m_trans_11 = vmull_n_s16(o_1, (WORD32)g_ai2_ihevc_trans_4[1][1]);
|
|
|
|
temp3 = vmlal_n_s16(o_1_m_trans_10, o_0, (WORD32)g_ai2_ihevc_trans_4[1][1]);
|
|
temp4 = vmlsl_n_s16(o_1_m_trans_11, o_0, (WORD32)g_ai2_ihevc_trans_4[1][0]);
|
|
|
|
transpose_s32_4x4(&temp1, &temp3, &temp2, &temp4);
|
|
|
|
e_03 = vaddq_s32(temp1, temp4);
|
|
e_12 = vaddq_s32(temp3, temp2);
|
|
o_03 = vsubq_s32(temp1, temp4);
|
|
o_12 = vsubq_s32(temp3, temp2);
|
|
|
|
e_0_a_e_1 = vaddq_s32(e_03, e_12);
|
|
e_0_s_e_1 = vsubq_s32(e_03, e_12);
|
|
|
|
temp1 = vmulq_n_s32(e_0_a_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
|
|
temp2 = vmulq_n_s32(e_0_s_e_1, (WORD32)g_ai2_ihevc_trans_4[0][0]);
|
|
|
|
o_1_m_trans_10 = vmulq_n_s32(o_03, (WORD32)g_ai2_ihevc_trans_4[1][0]);
|
|
o_1_m_trans_11 = vmulq_n_s32(o_03, (WORD32)g_ai2_ihevc_trans_4[1][1]);
|
|
|
|
temp3 = vmlaq_n_s32(o_1_m_trans_10, o_12, (WORD32)g_ai2_ihevc_trans_4[1][1]);
|
|
temp4 = vmlsq_n_s32(o_1_m_trans_11, o_12, (WORD32)g_ai2_ihevc_trans_4[1][0]);
|
|
|
|
out_0 = vrshrn_n_s32(temp1, 9);
|
|
out_1 = vrshrn_n_s32(temp3, 9);
|
|
out_2 = vrshrn_n_s32(temp2, 9);
|
|
out_3 = vrshrn_n_s32(temp4, 9);
|
|
|
|
vst1_s16(pi2_dst, out_0);
|
|
vst1_s16(pi2_dst + dst_strd, out_1);
|
|
vst1_s16(pi2_dst + 2 * dst_strd, out_2);
|
|
vst1_s16(pi2_dst + 3 * dst_strd, out_3);
|
|
|
|
return sad;
|
|
}
|
|
|
|
/**
|
|
*******************************************************************************
|
|
*
|
|
* @brief
|
|
* This function performs residue calculation and forward transform type 1
|
|
* on input pixels
|
|
*
|
|
* @par Description:
|
|
* Performs residue calculation by subtracting source and prediction and
|
|
* followed by forward transform
|
|
*
|
|
* @param[in] pu1_src
|
|
* Input 4x4 pixels
|
|
*
|
|
* @param[in] pu1_pred
|
|
* Prediction data
|
|
*
|
|
* @param[in] pi2_tmp
|
|
* Temporary buffer of size 4x4
|
|
*
|
|
* @param[out] pi2_dst
|
|
* Output 4x4 coefficients
|
|
*
|
|
* @param[in] src_strd
|
|
* Input stride
|
|
*
|
|
* @param[in] pred_strd
|
|
* Prediction Stride
|
|
*
|
|
* @param[in] dst_strd
|
|
* Output Stride
|
|
*
|
|
* @param[in] e_chroma_plane
|
|
* Enum singalling chroma plane
|
|
*
|
|
* @returns block sad
|
|
*
|
|
* @remarks
|
|
* None
|
|
*
|
|
*******************************************************************************
|
|
*/
|
|
UWORD32 ihevc_resi_trans_4x4_ttype1_neon(
|
|
UWORD8 *pu1_src,
|
|
UWORD8 *pu1_pred,
|
|
WORD32 *pi4_temp,
|
|
WORD16 *pi2_dst,
|
|
WORD32 src_strd,
|
|
WORD32 pred_strd,
|
|
WORD32 dst_strd,
|
|
CHROMA_PLANE_ID_T e_chroma_plane)
|
|
{
|
|
UWORD32 sad;
|
|
int16x4_t src0_4x16b;
|
|
int16x4_t src1_4x16b;
|
|
int16x4_t src2_4x16b;
|
|
int16x4_t src3_4x16b;
|
|
int32x4_t src0_4x32b;
|
|
int32x4_t src1_4x32b;
|
|
int32x4_t src2_4x32b;
|
|
int32x4_t src3_4x32b;
|
|
/*load source and pred values */
|
|
const uint8x16_t src_u8 = load_unaligned_u8q(pu1_src, src_strd);
|
|
const uint8x16_t pred_u8 = load_unaligned_u8q(pu1_pred, pred_strd);
|
|
|
|
const int16x8_t src_reg0 =
|
|
vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8)));
|
|
const int16x8_t src_reg1 =
|
|
vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(src_u8), vget_high_u8(pred_u8)));
|
|
|
|
int32x4_t add_val = vdupq_n_s32(1);
|
|
|
|
uint16x8_t abs = vabdl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8));
|
|
uint32x4_t b;
|
|
uint64x2_t c;
|
|
UNUSED(e_chroma_plane);
|
|
abs = vabal_u8(abs, vget_high_u8(src_u8), vget_high_u8(pred_u8));
|
|
b = vpaddlq_u16(abs);
|
|
c = vpaddlq_u32(b);
|
|
sad = vget_lane_u32(vadd_u32(vreinterpret_u32_u64(vget_low_u64(c)),
|
|
vreinterpret_u32_u64(vget_high_u64(c))),
|
|
0);
|
|
|
|
(void)pi4_temp;
|
|
|
|
/************************* 4x4 16bit Transpose ***********************/
|
|
src0_4x16b = vget_low_s16(src_reg0);
|
|
src1_4x16b = vget_high_s16(src_reg0);
|
|
src2_4x16b = vget_low_s16(src_reg1);
|
|
src3_4x16b = vget_high_s16(src_reg1);
|
|
|
|
transpose_s16_4x4d(&src0_4x16b, &src1_4x16b, &src2_4x16b, &src3_4x16b);
|
|
|
|
/************************** 4x4 Transpose End *************************/
|
|
|
|
/* Residue + Forward Transform 1st stage */
|
|
/* coeff2_4x32b = 74 74 74 74 */
|
|
const int32x4_t coeff2_4x32b =
|
|
vdupq_n_s32(74); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[2][0]);
|
|
/* coeff0_4x32b = 29 29 29 29 */
|
|
const int32x4_t coeff0_4x32b =
|
|
vdupq_n_s32(29); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[0][0]);
|
|
/* coeff1_4x32b = 55 55 55 55 */
|
|
const int32x4_t coeff1_4x32b =
|
|
vdupq_n_s32(55); //vld1q_s32(&g_ai4_ihevc_trans_dst_intr_4[1][0]);
|
|
|
|
/* c0 to c3 calculation */
|
|
int32x4_t c0_4x32b = vaddl_s16(src0_4x16b, src3_4x16b); /* r0+r3 */
|
|
int32x4_t c1_4x32b = vaddl_s16(src1_4x16b, src3_4x16b); /* r1+r3 */
|
|
int32x4_t c2_4x32b = vsubl_s16(src0_4x16b, src1_4x16b); /* r0-r1 */
|
|
int32x4_t c3_4x32b = vmulq_s32(vmovl_s16(src2_4x16b), coeff2_4x32b); /* 74*r2 */
|
|
src0_4x16b = vadd_s16(src0_4x16b, src1_4x16b); /* r0+r1 */
|
|
|
|
src1_4x32b = vsubl_s16(src0_4x16b, src3_4x16b); /* r0+r1-r3 */
|
|
src0_4x32b = vmlaq_s32(c3_4x32b, c0_4x32b, coeff0_4x32b); /* 29*c0 + c3 */
|
|
src2_4x32b = vmulq_s32(c2_4x32b, coeff0_4x32b); /* 29*c2 - c3 */
|
|
src3_4x32b = vmlaq_s32(c3_4x32b, c2_4x32b, coeff1_4x32b); /* 55*c2 + c3 */
|
|
src2_4x32b = vsubq_s32(src2_4x32b, c3_4x32b);
|
|
|
|
src0_4x32b = vmlaq_s32(src0_4x32b, c1_4x32b, coeff1_4x32b); /* 29*c0 + 55*c1 + c3 */
|
|
src2_4x32b = vmlaq_s32(src2_4x32b, c0_4x32b, coeff1_4x32b); /* 29*c2 + 55*c0 - c3 */
|
|
c1_4x32b = vmulq_s32(c1_4x32b, coeff0_4x32b); /* 55*c2 - 29*c1 + c3 */
|
|
src1_4x32b = vmulq_s32(src1_4x32b, coeff2_4x32b); /*74*(r0+r1-r3)*/
|
|
src3_4x32b = vsubq_s32(src3_4x32b, c1_4x32b);
|
|
|
|
/* result + add */
|
|
src1_4x32b = vaddq_s32(src1_4x32b, add_val);
|
|
src0_4x32b = vaddq_s32(src0_4x32b, add_val);
|
|
src2_4x32b = vaddq_s32(src2_4x32b, add_val);
|
|
src3_4x32b = vaddq_s32(src3_4x32b, add_val);
|
|
/* result >> shift */
|
|
src1_4x32b = vshrq_n_s32(src1_4x32b, 1);
|
|
src0_4x32b = vshrq_n_s32(src0_4x32b, 1);
|
|
src2_4x32b = vshrq_n_s32(src2_4x32b, 1);
|
|
src3_4x32b = vshrq_n_s32(src3_4x32b, 1);
|
|
/* Forward transform 2nd stage */
|
|
{
|
|
/************************* 4x4 32bit Transpose ***********************/
|
|
|
|
transpose_s32_4x4(&src0_4x32b, &src1_4x32b, &src2_4x32b, &src3_4x32b);
|
|
|
|
/************************** 4x4 Transpose End *************************/
|
|
|
|
/* add value */
|
|
add_val = vdupq_n_s32(128);
|
|
c0_4x32b = vaddq_s32(src0_4x32b, src3_4x32b); /* r0+r3 */
|
|
c1_4x32b = vaddq_s32(src1_4x32b, src3_4x32b); /* r1+r3 */
|
|
c2_4x32b = vsubq_s32(src0_4x32b, src1_4x32b); /* r0-r1 */
|
|
c3_4x32b = vmulq_s32(src2_4x32b, coeff2_4x32b); /* 74*r2 */
|
|
src1_4x32b = vaddq_s32(src0_4x32b, src1_4x32b); /* r0+r1 */
|
|
|
|
src1_4x32b = vsubq_s32(src1_4x32b, src3_4x32b); /* r0+r1-r3 */
|
|
src0_4x32b = vmlaq_s32(c3_4x32b, c0_4x32b, coeff0_4x32b); /* 29*c0 + c3 */
|
|
src2_4x32b = vmulq_s32(c2_4x32b, coeff0_4x32b); /* 29*c2 - c3 */
|
|
src3_4x32b = vmlaq_s32(c3_4x32b, c2_4x32b, coeff1_4x32b); /* 55*c2 + c3 */
|
|
src2_4x32b = vsubq_s32(src2_4x32b, c3_4x32b);
|
|
|
|
src0_4x32b = vmlaq_s32(src0_4x32b, c1_4x32b, coeff1_4x32b); /* 29*c0 + 55*c1 + c3 */
|
|
src2_4x32b = vmlaq_s32(src2_4x32b, c0_4x32b, coeff1_4x32b); /* 29*c2 + 55*c0 - c3 */
|
|
c1_4x32b = vmulq_s32(c1_4x32b, coeff0_4x32b); /* 55*c2 - 29*c1 + c3 */
|
|
src1_4x32b = vmulq_s32(src1_4x32b, coeff2_4x32b); /*74*(r0+r1-r3)*/
|
|
src3_4x32b = vsubq_s32(src3_4x32b, c1_4x32b);
|
|
|
|
/* result + add */
|
|
src1_4x32b = vaddq_s32(src1_4x32b, add_val);
|
|
src0_4x32b = vaddq_s32(src0_4x32b, add_val);
|
|
src2_4x32b = vaddq_s32(src2_4x32b, add_val);
|
|
src3_4x32b = vaddq_s32(src3_4x32b, add_val);
|
|
|
|
src1_4x32b = vshrq_n_s32(src1_4x32b, 8);
|
|
src0_4x32b = vshrq_n_s32(src0_4x32b, 8);
|
|
src2_4x32b = vshrq_n_s32(src2_4x32b, 8);
|
|
src3_4x32b = vshrq_n_s32(src3_4x32b, 8);
|
|
|
|
vst1_s16((pi2_dst + dst_strd), vmovn_s32(src1_4x32b));
|
|
vst1_s16(pi2_dst, vmovn_s32(src0_4x32b));
|
|
vst1_s16((pi2_dst + 2 * dst_strd), vmovn_s32(src2_4x32b));
|
|
vst1_s16((pi2_dst + 3 * dst_strd), vmovn_s32(src3_4x32b));
|
|
}
|
|
return sad;
|
|
}
|
|
|
|
/**
|
|
*******************************************************************************
|
|
*
|
|
* @brief
|
|
* This function performs residue calculation and forward transform on
|
|
* input pixels
|
|
*
|
|
* @par Description:
|
|
* Performs residue calculation by subtracting source and prediction and
|
|
* followed by forward transform
|
|
*
|
|
* @param[in] pu1_src
|
|
* Input 8x8 pixels
|
|
*
|
|
* @param[in] pu1_pred
|
|
* Prediction data
|
|
*
|
|
* @param[in] pi2_tmp
|
|
* Temporary buffer of size 8x8
|
|
*
|
|
* @param[out] pi2_dst
|
|
* Output 8x8 coefficients
|
|
*
|
|
* @param[in] src_strd
|
|
* Input stride
|
|
*
|
|
* @param[in] pred_strd
|
|
* Prediction Stride
|
|
*
|
|
* @param[in] dst_strd
|
|
* Output Stride
|
|
*
|
|
* @param[in] e_chroma_plane
|
|
* Enum singalling chroma plane
|
|
*
|
|
* @returns Void
|
|
*
|
|
* @remarks
|
|
* None
|
|
*
|
|
*******************************************************************************
|
|
*/
|
|
UWORD32 ihevc_resi_trans_8x8_neon(
|
|
UWORD8 *pu1_src,
|
|
UWORD8 *pu1_pred,
|
|
WORD32 *pi4_temp,
|
|
WORD16 *pi2_dst,
|
|
WORD32 src_strd,
|
|
WORD32 pred_strd,
|
|
WORD32 dst_strd,
|
|
CHROMA_PLANE_ID_T e_chroma_plane)
|
|
{
|
|
int16x8_t diff_16[8];
|
|
int16x8_t abs = vdupq_n_s16(0);
|
|
int32x4_t tmp_a;
|
|
int64x2_t tmp_b;
|
|
int32x2_t sad_v;
|
|
int32x4x2_t a0, a1, a2, a3, a4, a5, a6, a7;
|
|
UWORD32 sad;
|
|
|
|
(void)pi4_temp;
|
|
#define RESIDUE(k) \
|
|
if(NULL_PLANE == e_chroma_plane) \
|
|
{ \
|
|
const uint8x8_t s##k = vld1_u8(pu1_src); \
|
|
const uint8x8_t p##k = vld1_u8(pu1_pred); \
|
|
diff_16[k] = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \
|
|
pu1_src += src_strd; \
|
|
pu1_pred += pred_strd; \
|
|
abs = vaddq_s16(abs, vabsq_s16(diff_16[k])); \
|
|
} \
|
|
else \
|
|
{ \
|
|
const uint8x8_t s##k = vld2_u8(pu1_src).val[e_chroma_plane]; \
|
|
const uint8x8_t p##k = vld2_u8(pu1_pred).val[e_chroma_plane]; \
|
|
diff_16[k] = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \
|
|
pu1_src += src_strd; \
|
|
pu1_pred += pred_strd; \
|
|
abs = vaddq_s16(abs, vabsq_s16(diff_16[k])); \
|
|
}
|
|
|
|
// stage 1
|
|
RESIDUE(0);
|
|
RESIDUE(1);
|
|
RESIDUE(2);
|
|
RESIDUE(3);
|
|
RESIDUE(4);
|
|
RESIDUE(5);
|
|
RESIDUE(6);
|
|
RESIDUE(7);
|
|
|
|
tmp_a = vpaddlq_s16(abs);
|
|
tmp_b = vpaddlq_s32(tmp_a);
|
|
sad_v = vadd_s32(vreinterpret_s32_s64(vget_low_s64(tmp_b)),
|
|
vreinterpret_s32_s64(vget_high_s64(tmp_b)));
|
|
sad = vget_lane_s32(sad_v, 0);
|
|
|
|
transpose_s16_8x8(
|
|
&diff_16[0],
|
|
&diff_16[1],
|
|
&diff_16[2],
|
|
&diff_16[3],
|
|
&diff_16[4],
|
|
&diff_16[5],
|
|
&diff_16[6],
|
|
&diff_16[7]);
|
|
|
|
{
|
|
const int16x8_t o3 = vsubq_s16(diff_16[3], diff_16[4]); /*C3 - C4*/
|
|
const int16x8_t o2 = vsubq_s16(diff_16[2], diff_16[5]); /*C2 - C5*/
|
|
const int16x8_t o1 = vsubq_s16(diff_16[1], diff_16[6]); /*C1 - C6*/
|
|
const int16x8_t o0 = vsubq_s16(diff_16[0], diff_16[7]); /*C0 - C7*/
|
|
const int16x8_t e0 = vaddq_s16(diff_16[0], diff_16[7]); /*C0 + C7*/
|
|
const int16x8_t e1 = vaddq_s16(diff_16[1], diff_16[6]); /*C1 + C6*/
|
|
const int16x8_t e2 = vaddq_s16(diff_16[2], diff_16[5]); /*C2 + C5*/
|
|
const int16x8_t e3 = vaddq_s16(diff_16[3], diff_16[4]); /*C3 + C4*/
|
|
|
|
const int16x8_t ee0 = vaddq_s16(e0, e3); /*C0 + C3 + C4 + C7*/
|
|
const int16x8_t ee1 = vaddq_s16(e1, e2); /*C1 + C2 + C5 + C6*/
|
|
const int16x8_t eo0 = vsubq_s16(e0, e3); /*C0 - C3 - C4 + C7*/
|
|
const int16x8_t eo1 = vsubq_s16(e1, e2); /*C1 - C2 - C5 + C6*/
|
|
|
|
/*C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7*/
|
|
const int16x8_t eee = vaddq_s16(ee1, ee0);
|
|
/*C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7*/
|
|
const int16x8_t eeo = vsubq_s16(ee0, ee1);
|
|
|
|
/*F2[0] of 83*(C0 - C3 - C4 + C7)*/
|
|
a2.val[0] = vmull_n_s16(vget_low_s16(eo0), 83);
|
|
/*F6[0] of 36*(C0 - C3 - C4 + C7)*/
|
|
a6.val[0] = vmull_n_s16(vget_low_s16(eo0), 36);
|
|
/*F2[1] of 83*(C0 - C3 - C4 + C7)*/
|
|
a2.val[1] = vmull_n_s16(vget_high_s16(eo0), 83);
|
|
/*F6[1] of 36*(C0 - C3 - C4 + C7)*/
|
|
a6.val[1] = vmull_n_s16(vget_high_s16(eo0), 36);
|
|
|
|
/*F6[1] = 36*(C0 - C3 - C4 + C7) - 83*(C1 - C2 - C5 + C6)*/
|
|
a6.val[1] = vmlsl_n_s16(a6.val[1], vget_high_s16(eo1), 83);
|
|
/*F2[1] = 83*(C0 - C3 - C4 + C7) + 36*(C1 - C2 - C5 + C6)*/
|
|
a2.val[1] = vmlal_n_s16(a2.val[1], vget_high_s16(eo1), 36);
|
|
/*F6[0] = 36*(C0 - C3 - C4 + C7) - 83*(C1 - C2 - C5 + C6)*/
|
|
a6.val[0] = vmlsl_n_s16(a6.val[0], vget_low_s16(eo1), 83);
|
|
/*F2[0] = 83*(C0 - C3 - C4 + C7) + 36*(C1 - C2 - C5 + C6)*/
|
|
a2.val[0] = vmlal_n_s16(a2.val[0], vget_low_s16(eo1), 36);
|
|
|
|
/*F0[0] = 64*(C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7)*/
|
|
a0.val[0] = vshll_n_s16(vget_low_s16(eee), 6);
|
|
/*F0[1] = 64*(C0 + C1 + C2 + C3 + C4 + C5 + C6 + C7)*/
|
|
a0.val[1] = vshll_n_s16(vget_high_s16(eee), 6);
|
|
/*F4[0] = 64*(C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7)*/
|
|
a4.val[0] = vshll_n_s16(vget_low_s16(eeo), 6);
|
|
/*F4[1] = 64*(C0 - C1 - C2 + C3 + C4 - C5 - C6 + C7)*/
|
|
a4.val[1] = vshll_n_s16(vget_high_s16(eeo), 6);
|
|
|
|
a7.val[0] = vmull_n_s16(vget_low_s16(o0), 18); /*F7[0] = 18*(C0 - C7)*/
|
|
a5.val[0] = vmull_n_s16(vget_low_s16(o0), 50); /*F5[0] = 50*(C0 - C7)*/
|
|
a3.val[0] = vmull_n_s16(vget_low_s16(o0), 75); /*F3[0] = 75*(C0 - C7)*/
|
|
a1.val[0] = vmull_n_s16(vget_low_s16(o0), 89); /*F1[0] = 89*(C0 - C7)*/
|
|
a1.val[1] = vmull_n_s16(vget_high_s16(o0), 89); /*F1[1] = 89*(C0 - C7)*/
|
|
a3.val[1] = vmull_n_s16(vget_high_s16(o0), 75); /*F3[1] = 75*(C0 - C7)*/
|
|
a5.val[1] = vmull_n_s16(vget_high_s16(o0), 50); /*F5[1] = 50*(C0 - C7)*/
|
|
a7.val[1] = vmull_n_s16(vget_high_s16(o0), 18); /*F7[1] = 18*(C0 - C7)*/
|
|
|
|
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6)*/
|
|
a7.val[0] = vmlsl_n_s16(a7.val[0], vget_low_s16(o1), 50);
|
|
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6)*/
|
|
a5.val[0] = vmlsl_n_s16(a5.val[0], vget_low_s16(o1), 89);
|
|
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6)*/
|
|
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o1), 18);
|
|
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6)*/
|
|
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o1), 75);
|
|
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6)*/
|
|
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o1), 75);
|
|
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6)*/
|
|
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o1), 18);
|
|
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6)*/
|
|
a5.val[1] = vmlsl_n_s16(a5.val[1], vget_high_s16(o1), 89);
|
|
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6)*/
|
|
a7.val[1] = vmlsl_n_s16(a7.val[1], vget_high_s16(o1), 50);
|
|
|
|
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5)*/
|
|
a7.val[0] = vmlal_n_s16(a7.val[0], vget_low_s16(o2), 75);
|
|
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5)*/
|
|
a5.val[0] = vmlal_n_s16(a5.val[0], vget_low_s16(o2), 18);
|
|
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5)*/
|
|
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o2), 89);
|
|
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5)*/
|
|
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o2), 50);
|
|
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5)*/
|
|
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o2), 50);
|
|
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5)*/
|
|
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o2), 89);
|
|
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5)*/
|
|
a5.val[1] = vmlal_n_s16(a5.val[1], vget_high_s16(o2), 18);
|
|
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5)*/
|
|
a7.val[1] = vmlal_n_s16(a7.val[1], vget_high_s16(o2), 75);
|
|
|
|
/*F7[0] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5) - 89*(C3 - C4)*/
|
|
a7.val[0] = vmlsl_n_s16(a7.val[0], vget_low_s16(o3), 89);
|
|
/*F5[0] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5) + 75*(C3 - C4)*/
|
|
a5.val[0] = vmlal_n_s16(a5.val[0], vget_low_s16(o3), 75);
|
|
/*F3[0] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5) - 50*(C3 - C4)*/
|
|
a3.val[0] = vmlsl_n_s16(a3.val[0], vget_low_s16(o3), 50);
|
|
/*F1[0] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5) + 18*(C3 - C4)*/
|
|
a1.val[0] = vmlal_n_s16(a1.val[0], vget_low_s16(o3), 18);
|
|
/*F1[1] = 89*(C0 - C7) + 75*(C1 - C6) + 50*(C2 - C5) + 18*(C3 - C4)*/
|
|
a1.val[1] = vmlal_n_s16(a1.val[1], vget_high_s16(o3), 18);
|
|
/*F3[1] = 75*(C0 - C7) - 18*(C1 - C6) - 89*(C2 - C5) - 50*(C3 - C4)*/
|
|
a3.val[1] = vmlsl_n_s16(a3.val[1], vget_high_s16(o3), 50);
|
|
/*F5[1] = 50*(C0 - C7) - 89*(C1 - C6) + 18*(C2 - C5) + 75*(C3 - C4)*/
|
|
a5.val[1] = vmlal_n_s16(a5.val[1], vget_high_s16(o3), 75);
|
|
/*F7[1] = 18*(C0 - C7) - 50*(C1 - C6) + 75*(C2 - C5) - 89*(C3 - C4)*/
|
|
a7.val[1] = vmlsl_n_s16(a7.val[1], vget_high_s16(o3), 89);
|
|
}
|
|
|
|
//Stage 2
|
|
{
|
|
int32x4_t h0, h1, h2, h3, h4, h5, h6, h7;
|
|
int32x4_t e0_2, e1_2, e2_2, e3_2;
|
|
int32x4_t o0_2, o1_2, o2_2, o3_2;
|
|
int32x4_t ee1_2, eo1_2, eo0_2, ee0_2;
|
|
int16x4_t row0, row1, row2, row3, row4, row5, row6, row7;
|
|
|
|
/*Transposing second half of transform stage 1 (1)*/
|
|
int32x4x2_t b1 = vtrnq_s32(a0.val[1], a1.val[1]);
|
|
int32x4x2_t b3 = vtrnq_s32(a2.val[1], a3.val[1]);
|
|
int32x4x2_t b0 = vtrnq_s32(a0.val[0], a1.val[0]);
|
|
int32x4x2_t b2 = vtrnq_s32(a2.val[0], a3.val[0]);
|
|
|
|
/*Transposing second half of transform stage 1 (2)*/
|
|
a0.val[0] = vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b2.val[0]));
|
|
a2.val[0] = vcombine_s32(vget_high_s32(b0.val[0]), vget_high_s32(b2.val[0]));
|
|
a1.val[0] = vcombine_s32(vget_low_s32(b0.val[1]), vget_low_s32(b2.val[1]));
|
|
a3.val[0] = vcombine_s32(vget_high_s32(b0.val[1]), vget_high_s32(b2.val[1]));
|
|
a0.val[1] = vcombine_s32(vget_low_s32(b1.val[0]), vget_low_s32(b3.val[0]));
|
|
a2.val[1] = vcombine_s32(vget_high_s32(b1.val[0]), vget_high_s32(b3.val[0]));
|
|
a1.val[1] = vcombine_s32(vget_low_s32(b1.val[1]), vget_low_s32(b3.val[1]));
|
|
a3.val[1] = vcombine_s32(vget_high_s32(b1.val[1]), vget_high_s32(b3.val[1]));
|
|
|
|
o0_2 = vsubq_s32(a0.val[0], a3.val[1]); /*B0 - B7*/
|
|
o1_2 = vsubq_s32(a1.val[0], a2.val[1]); /*B1 - B6*/
|
|
o2_2 = vsubq_s32(a2.val[0], a1.val[1]); /*B2 - B5*/
|
|
o3_2 = vsubq_s32(a3.val[0], a0.val[1]); /*B3 - B4*/
|
|
e3_2 = vaddq_s32(a3.val[0], a0.val[1]); /*B3 + B4*/
|
|
e2_2 = vaddq_s32(a2.val[0], a1.val[1]); /*B2 + B5*/
|
|
e1_2 = vaddq_s32(a1.val[0], a2.val[1]); /*B1 + B6*/
|
|
e0_2 = vaddq_s32(a0.val[0], a3.val[1]); /*B0 + B7*/
|
|
|
|
eo1_2 = vsubq_s32(e1_2, e2_2); /*B1 - B2 - B5 + B6*/
|
|
ee1_2 = vaddq_s32(e1_2, e2_2); /*B1 + B2 + B5 + B6*/
|
|
eo0_2 = vsubq_s32(e0_2, e3_2); /*B0 - B3 - B4 + B7*/
|
|
ee0_2 = vaddq_s32(e0_2, e3_2); /*B0 + B3 + B4 + B7*/
|
|
|
|
/* F4 = B0 - B1 - B2 + B3 + B4 - B5 - B6 + B7*/
|
|
h4 = vsubq_s32(ee0_2, ee1_2);
|
|
/* F0 = B0 + B1 + B2 + B3 + B4 + B5 + B6 + B7*/
|
|
h0 = vaddq_s32(ee0_2, ee1_2);
|
|
/* Truncating last 11 bits in H0*/
|
|
row0 = vrshrn_n_s32(h0, 5);
|
|
/*First half-row of row 1 of transform stage 2 (H0) stored*/
|
|
vst1_s16(pi2_dst, row0);
|
|
/* Truncating last 11 bits in H4*/
|
|
row4 = vrshrn_n_s32(h4, 5);
|
|
/*First half-row of row 5 of transform stage 2 (H4) stored*/
|
|
vst1_s16(pi2_dst + 4 * dst_strd, row4);
|
|
|
|
/* F6 = 36*(B0 - B3 - B4 + B7) */
|
|
h6 = vmulq_n_s32(eo0_2, 36);
|
|
/* F2 = 83*(B0 - B3 - B4 + B7) */
|
|
h2 = vmulq_n_s32(eo0_2, 83);
|
|
/*H2 = 83*(B0 - B3 - B4 + B7) + 36*(B1 - B2 - B5 + B6)*/
|
|
h2 = vmlaq_n_s32(h2, eo1_2, 36);
|
|
/*H6 = 36*(B0 - B3 - B4 + B7) - 83*(B1 - B2 - B5 + B6)*/
|
|
h6 = vmlsq_n_s32(h6, eo1_2, 83);
|
|
/* Truncating last 11 bits in H6*/
|
|
row6 = vrshrn_n_s32(h6, 11);
|
|
/*First half-row of row 7 of transform stage 2 (H6) stored*/
|
|
vst1_s16(pi2_dst + 6 * dst_strd, row6);
|
|
/* Truncating last 11 bits in H2*/
|
|
row2 = vrshrn_n_s32(h2, 11);
|
|
/*First half-row of row 3 of transform stage 2 (H2) stored*/
|
|
vst1_s16(pi2_dst + 2 * dst_strd, row2);
|
|
|
|
h1 = vmulq_n_s32(o0_2, 89); /* H1 = 89*(B0 - B7) */
|
|
h3 = vmulq_n_s32(o0_2, 75); /* H3 = 75*(B0 - B7) */
|
|
h5 = vmulq_n_s32(o0_2, 50); /* H5 = 50*(B0 - B7) */
|
|
h7 = vmulq_n_s32(o0_2, 18); /* H7 = 18*(B0 - B7) */
|
|
|
|
h7 = vmlsq_n_s32(h7, o1_2, 50); /* H7 = 18*(B0 - B7) - 50*(B1 - B6) */
|
|
h5 = vmlsq_n_s32(h5, o1_2, 89); /* H5 = 50*(B0 - B7) - 89*(B1 - B6) */
|
|
h3 = vmlsq_n_s32(h3, o1_2, 18); /* H3 = 75*(B0 - B7) - 18*(B1 - B6) */
|
|
h1 = vmlaq_n_s32(h1, o1_2, 75); /* H1 = 89*(B0 - B7) + 75*(B1 - B6) */
|
|
|
|
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) */
|
|
h1 = vmlaq_n_s32(h1, o2_2, 50);
|
|
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) */
|
|
h3 = vmlsq_n_s32(h3, o2_2, 89);
|
|
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) */
|
|
h5 = vmlaq_n_s32(h5, o2_2, 18);
|
|
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) */
|
|
h7 = vmlaq_n_s32(h7, o2_2, 75);
|
|
|
|
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) - 89*(B3 - B4) */
|
|
h7 = vmlsq_n_s32(h7, o3_2, 89);
|
|
/* Truncating last 11 bits in H7*/
|
|
row7 = vrshrn_n_s32(h7, 11);
|
|
/*First half-row of row 8 of transform stage 2 (H7) stored*/
|
|
vst1_s16(pi2_dst + 7 * dst_strd, row7);
|
|
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) + 75*(B3 - B4) */
|
|
h5 = vmlaq_n_s32(h5, o3_2, 75);
|
|
/* Truncating last 11 bits in H5*/
|
|
row5 = vrshrn_n_s32(h5, 11);
|
|
/*First half-row of row 6 of transform stage 2 (H5) stored*/
|
|
vst1_s16(pi2_dst + 5 * dst_strd, row5);
|
|
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) - 50*(B3 - B4) */
|
|
h3 = vmlsq_n_s32(h3, o3_2, 50);
|
|
/* Truncating last 11 bits in H3*/
|
|
row3 = vrshrn_n_s32(h3, 11);
|
|
/*First half-row of row 4 of transform stage 2 (H3) stored*/
|
|
vst1_s16(pi2_dst + 3 * dst_strd, row3);
|
|
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) + 18*(B3 - B4) */
|
|
h1 = vmlaq_n_s32(h1, o3_2, 18);
|
|
/* Truncating last 11 bits in H1*/
|
|
row1 = vrshrn_n_s32(h1, 11);
|
|
/*First half-row of row 2 of transform stage 2 (H1) stored*/
|
|
vst1_s16(pi2_dst + dst_strd, row1);
|
|
}
|
|
|
|
pi2_dst += 4;
|
|
|
|
{
|
|
int32x4_t h0, h1, h2, h3, h4, h5, h6, h7;
|
|
int32x4_t e0_2, e1_2, e2_2, e3_2;
|
|
int32x4_t o0_2, o1_2, o2_2, o3_2;
|
|
int32x4_t ee1_2, eo1_2, eo0_2, ee0_2;
|
|
int16x4_t row0, row1, row2, row3, row4, row5, row6, row7;
|
|
|
|
/*Transposing second half of transform stage 1 (1)*/
|
|
int32x4x2_t b1 = vtrnq_s32(a4.val[1], a5.val[1]);
|
|
int32x4x2_t b3 = vtrnq_s32(a6.val[1], a7.val[1]);
|
|
int32x4x2_t b0 = vtrnq_s32(a4.val[0], a5.val[0]);
|
|
int32x4x2_t b2 = vtrnq_s32(a6.val[0], a7.val[0]);
|
|
|
|
/*Transposing second half of transform stage 1 (2)*/
|
|
a0.val[0] = vcombine_s32(vget_low_s32(b0.val[0]), vget_low_s32(b2.val[0]));
|
|
a2.val[0] = vcombine_s32(vget_high_s32(b0.val[0]), vget_high_s32(b2.val[0]));
|
|
a1.val[0] = vcombine_s32(vget_low_s32(b0.val[1]), vget_low_s32(b2.val[1]));
|
|
a3.val[0] = vcombine_s32(vget_high_s32(b0.val[1]), vget_high_s32(b2.val[1]));
|
|
a0.val[1] = vcombine_s32(vget_low_s32(b1.val[0]), vget_low_s32(b3.val[0]));
|
|
a2.val[1] = vcombine_s32(vget_high_s32(b1.val[0]), vget_high_s32(b3.val[0]));
|
|
a1.val[1] = vcombine_s32(vget_low_s32(b1.val[1]), vget_low_s32(b3.val[1]));
|
|
a3.val[1] = vcombine_s32(vget_high_s32(b1.val[1]), vget_high_s32(b3.val[1]));
|
|
|
|
o0_2 = vsubq_s32(a0.val[0], a3.val[1]); /*B0 - B7*/
|
|
o1_2 = vsubq_s32(a1.val[0], a2.val[1]); /*B1 - B6*/
|
|
o2_2 = vsubq_s32(a2.val[0], a1.val[1]); /*B2 - B5*/
|
|
o3_2 = vsubq_s32(a3.val[0], a0.val[1]); /*B3 - B4*/
|
|
e3_2 = vaddq_s32(a3.val[0], a0.val[1]); /*B3 + B4*/
|
|
e2_2 = vaddq_s32(a2.val[0], a1.val[1]); /*B2 + B5*/
|
|
e1_2 = vaddq_s32(a1.val[0], a2.val[1]); /*B1 + B6*/
|
|
e0_2 = vaddq_s32(a0.val[0], a3.val[1]); /*B0 + B7*/
|
|
|
|
eo1_2 = vsubq_s32(e1_2, e2_2); /*B1 - B2 - B5 + B6*/
|
|
ee1_2 = vaddq_s32(e1_2, e2_2); /*B1 + B2 + B5 + B6*/
|
|
eo0_2 = vsubq_s32(e0_2, e3_2); /*B0 - B3 - B4 + B7*/
|
|
ee0_2 = vaddq_s32(e0_2, e3_2); /*B0 + B3 + B4 + B7*/
|
|
|
|
/* F4 = B0 - B1 - B2 + B3 + B4 - B5 - B6 + B7*/
|
|
h4 = vsubq_s32(ee0_2, ee1_2);
|
|
/* F0 = B0 + B1 + B2 + B3 + B4 + B5 + B6 + B7*/
|
|
h0 = vaddq_s32(ee0_2, ee1_2);
|
|
/* Truncating last 11 bits in H0*/
|
|
row0 = vrshrn_n_s32(h0, 5);
|
|
/*First half-row of row 1 of transform stage 2 (H0) stored*/
|
|
vst1_s16(pi2_dst, row0);
|
|
/* Truncating last 11 bits in H4*/
|
|
row4 = vrshrn_n_s32(h4, 5);
|
|
/*First half-row of row 5 of transform stage 2 (H4) stored*/
|
|
vst1_s16(pi2_dst + 4 * dst_strd, row4);
|
|
|
|
/* F6 = 36*(B0 - B3 - B4 + B7) */
|
|
h6 = vmulq_n_s32(eo0_2, 36);
|
|
/* F2 = 83*(B0 - B3 - B4 + B7) */
|
|
h2 = vmulq_n_s32(eo0_2, 83);
|
|
/*H2 = 83*(B0 - B3 - B4 + B7) + 36*(B1 - B2 - B5 + B6)*/
|
|
h2 = vmlaq_n_s32(h2, eo1_2, 36);
|
|
/*H6 = 36*(B0 - B3 - B4 + B7) - 83*(B1 - B2 - B5 + B6)*/
|
|
h6 = vmlsq_n_s32(h6, eo1_2, 83);
|
|
/* Truncating last 11 bits in H6*/
|
|
row6 = vrshrn_n_s32(h6, 11);
|
|
/*First half-row of row 7 of transform stage 2 (H6) stored*/
|
|
vst1_s16(pi2_dst + 6 * dst_strd, row6);
|
|
/* Truncating last 11 bits in H2*/
|
|
row2 = vrshrn_n_s32(h2, 11);
|
|
/*First half-row of row 3 of transform stage 2 (H2) stored*/
|
|
vst1_s16(pi2_dst + 2 * dst_strd, row2);
|
|
|
|
h1 = vmulq_n_s32(o0_2, 89); /* H1 = 89*(B0 - B7) */
|
|
h3 = vmulq_n_s32(o0_2, 75); /* H3 = 75*(B0 - B7) */
|
|
h5 = vmulq_n_s32(o0_2, 50); /* H5 = 50*(B0 - B7) */
|
|
h7 = vmulq_n_s32(o0_2, 18); /* H7 = 18*(B0 - B7) */
|
|
|
|
h7 = vmlsq_n_s32(h7, o1_2, 50); /* H7 = 18*(B0 - B7) - 50*(B1 - B6) */
|
|
h5 = vmlsq_n_s32(h5, o1_2, 89); /* H5 = 50*(B0 - B7) - 89*(B1 - B6) */
|
|
h3 = vmlsq_n_s32(h3, o1_2, 18); /* H3 = 75*(B0 - B7) - 18*(B1 - B6) */
|
|
h1 = vmlaq_n_s32(h1, o1_2, 75); /* H1 = 89*(B0 - B7) + 75*(B1 - B6) */
|
|
|
|
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) */
|
|
h1 = vmlaq_n_s32(h1, o2_2, 50);
|
|
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) */
|
|
h3 = vmlsq_n_s32(h3, o2_2, 89);
|
|
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) */
|
|
h5 = vmlaq_n_s32(h5, o2_2, 18);
|
|
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) */
|
|
h7 = vmlaq_n_s32(h7, o2_2, 75);
|
|
|
|
/* H7 = 18*(B0 - B7) - 50*(B1 - B6) + 75*(B2 - B5) - 89*(B3 - B4) */
|
|
h7 = vmlsq_n_s32(h7, o3_2, 89);
|
|
/* Truncating last 11 bits in H7*/
|
|
row7 = vrshrn_n_s32(h7, 11);
|
|
/*First half-row of row 8 of transform stage 2 (H7) stored*/
|
|
vst1_s16(pi2_dst + 7 * dst_strd, row7);
|
|
/* H5 = 50*(B0 - B7) - 89*(B1 - B6) + 18*(B2 - B5) + 75*(B3 - B4) */
|
|
h5 = vmlaq_n_s32(h5, o3_2, 75);
|
|
/* Truncating last 11 bits in H5*/
|
|
row5 = vrshrn_n_s32(h5, 11);
|
|
/*First half-row of row 6 of transform stage 2 (H5) stored*/
|
|
vst1_s16(pi2_dst + 5 * dst_strd, row5);
|
|
/* H3 = 75*(B0 - B7) - 18*(B1 - B6) - 89*(B2 - B5) - 50*(B3 - B4) */
|
|
h3 = vmlsq_n_s32(h3, o3_2, 50);
|
|
/* Truncating last 11 bits in H3*/
|
|
row3 = vrshrn_n_s32(h3, 11);
|
|
/*First half-row of row 4 of transform stage 2 (H3) stored*/
|
|
vst1_s16(pi2_dst + 3 * dst_strd, row3);
|
|
/* H1 = 89*(B0 - B7) + 75*(B1 - B6) + 50*(B2 - B5) + 18*(B3 - B4) */
|
|
h1 = vmlaq_n_s32(h1, o3_2, 18);
|
|
/* Truncating last 11 bits in H1*/
|
|
row1 = vrshrn_n_s32(h1, 11);
|
|
/*First half-row of row 2 of transform stage 2 (H1) stored*/
|
|
vst1_s16(pi2_dst + dst_strd, row1);
|
|
}
|
|
return sad;
|
|
}
|
|
|
|
static INLINE void load(const uint8_t *a, int stride, uint8x8_t *b,
|
|
CHROMA_PLANE_ID_T e_chroma_plane)
|
|
{
|
|
int i;
|
|
|
|
if(e_chroma_plane == NULL_PLANE)
|
|
{
|
|
for (i = 0; i < 16; i++)
|
|
{
|
|
b[i] = vld1_u8(a);
|
|
a += stride;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (i = 0; i < 16; i++)
|
|
{
|
|
b[i] = vld2_u8(a).val[e_chroma_plane];
|
|
a += stride;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Store 8 16x8 values, assuming stride == 16.
|
|
static INLINE void store(WORD16 *a, int16x8_t *b /*[8]*/)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < 8; i++)
|
|
{
|
|
vst1q_s16(a, b[i]);
|
|
a += 16;
|
|
}
|
|
}
|
|
|
|
static INLINE void cross_input_16(int16x8_t *a /*[16]*/, int16x8_t *b /*[16]*/)
|
|
{
|
|
b[0] = vaddq_s16(a[0], a[15]);
|
|
b[1] = vaddq_s16(a[1], a[14]);
|
|
b[2] = vaddq_s16(a[2], a[13]);
|
|
b[3] = vaddq_s16(a[3], a[12]);
|
|
b[4] = vaddq_s16(a[4], a[11]);
|
|
b[5] = vaddq_s16(a[5], a[10]);
|
|
b[6] = vaddq_s16(a[6], a[9]);
|
|
b[7] = vaddq_s16(a[7], a[8]);
|
|
|
|
b[8] = vsubq_s16(a[7], a[8]);
|
|
b[9] = vsubq_s16(a[6], a[9]);
|
|
b[10] = vsubq_s16(a[5], a[10]);
|
|
b[11] = vsubq_s16(a[4], a[11]);
|
|
b[12] = vsubq_s16(a[3], a[12]);
|
|
b[13] = vsubq_s16(a[2], a[13]);
|
|
b[14] = vsubq_s16(a[1], a[14]);
|
|
b[15] = vsubq_s16(a[0], a[15]);
|
|
}
|
|
|
|
static INLINE void cross_input_32(int32x4x2_t *a /*[16][2]*/, int32x4x2_t *b /*[16][2]*/)
|
|
{
|
|
WORD32 i;
|
|
for(i = 0; i < 2; i++)
|
|
{
|
|
b[0].val[i] = vaddq_s32(a[0].val[i], a[15].val[i]);
|
|
b[1].val[i] = vaddq_s32(a[1].val[i], a[14].val[i]);
|
|
b[2].val[i] = vaddq_s32(a[2].val[i], a[13].val[i]);
|
|
b[3].val[i] = vaddq_s32(a[3].val[i], a[12].val[i]);
|
|
b[4].val[i] = vaddq_s32(a[4].val[i], a[11].val[i]);
|
|
b[5].val[i] = vaddq_s32(a[5].val[i], a[10].val[i]);
|
|
b[6].val[i] = vaddq_s32(a[6].val[i], a[9].val[i]);
|
|
b[7].val[i] = vaddq_s32(a[7].val[i], a[8].val[i]);
|
|
|
|
b[8].val[i] = vsubq_s32(a[7].val[i], a[8].val[i]);
|
|
b[9].val[i] = vsubq_s32(a[6].val[i], a[9].val[i]);
|
|
b[10].val[i] = vsubq_s32(a[5].val[i], a[10].val[i]);
|
|
b[11].val[i] = vsubq_s32(a[4].val[i], a[11].val[i]);
|
|
b[12].val[i] = vsubq_s32(a[3].val[i], a[12].val[i]);
|
|
b[13].val[i] = vsubq_s32(a[2].val[i], a[13].val[i]);
|
|
b[14].val[i] = vsubq_s32(a[1].val[i], a[14].val[i]);
|
|
b[15].val[i] = vsubq_s32(a[0].val[i], a[15].val[i]);
|
|
}
|
|
}
|
|
|
|
static INLINE int32x4_t diff(uint8x8_t *a /*[16]*/, uint8x8_t *b /*[16]*/, int16x8_t *c /*[16]*/)
|
|
{
|
|
int i;
|
|
int16x8_t abs = vdupq_n_s16(0);
|
|
|
|
for (i = 0; i < 16; i++)
|
|
{
|
|
c[i] = vreinterpretq_s16_u16(vsubl_u8(a[i], b[i]));
|
|
abs = vaddq_s16(abs, vabsq_s16(c[i]));
|
|
}
|
|
return vpaddlq_s16(abs);
|
|
}
|
|
|
|
static INLINE void partial_round_shift(int32x4x2_t *a, int16x8_t *b /*[16]*/)
|
|
{
|
|
WORD32 shift = 13, add;
|
|
add = 1 << (shift - 1);
|
|
|
|
const int32x4_t vecadd = vdupq_n_s32(add);
|
|
b[0] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[0].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[0].val[1], vecadd), 13));
|
|
b[1] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[1].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[1].val[1], vecadd), 13));
|
|
b[2] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[2].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[2].val[1], vecadd), 13));
|
|
b[3] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[3].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[3].val[1], vecadd), 13));
|
|
b[4] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[4].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[4].val[1], vecadd), 13));
|
|
b[5] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[5].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[5].val[1], vecadd), 13));
|
|
b[6] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[6].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[6].val[1], vecadd), 13));
|
|
b[7] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[7].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[7].val[1], vecadd), 13));
|
|
b[8] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[8].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[8].val[1], vecadd), 13));
|
|
b[9] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[9].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[9].val[1], vecadd), 13));
|
|
b[10] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[10].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[10].val[1], vecadd), 13));
|
|
b[11] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[11].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[11].val[1], vecadd), 13));
|
|
b[12] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[12].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[12].val[1], vecadd), 13));
|
|
b[13] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[13].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[13].val[1], vecadd), 13));
|
|
b[14] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[14].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[14].val[1], vecadd), 13));
|
|
b[15] = vcombine_s16(
|
|
vshrn_n_s32(vaddq_s32(a[15].val[0], vecadd), 13),
|
|
vshrn_n_s32(vaddq_s32(a[15].val[1], vecadd), 13));
|
|
}
|
|
|
|
static INLINE int32x4_t
|
|
add4(int32x4_t row1_low, int32x4_t row1_high, int32x4_t row2_low, int32x4_t row2_high)
|
|
{
|
|
int32x4_t sum1, sum2;
|
|
sum1 = vaddq_s32(row1_low, row1_high);
|
|
sum2 = vaddq_s32(row2_low, row2_high);
|
|
return vaddq_s32(sum1, sum2);
|
|
}
|
|
|
|
static INLINE void butterfly_one_coeff_16_32(
|
|
int16x8_t a, int16x8_t b, int16_t c, int32x4x2_t *row1, int32x4x2_t *row2)
|
|
{
|
|
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c);
|
|
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c);
|
|
//printf("multiply done\n");
|
|
row1->val[0] = vmlal_n_s16(a0, vget_low_s16(b), c);
|
|
row1->val[1] = vmlal_n_s16(a1, vget_high_s16(b), c);
|
|
row2->val[0] = vmlsl_n_s16(a0, vget_low_s16(b), c);
|
|
row2->val[1] = vmlsl_n_s16(a1, vget_high_s16(b), c);
|
|
}
|
|
|
|
static INLINE void butterfly_two_coeff_16_32(
|
|
int16x8_t a, int16x8_t b, int16_t c0, int16_t c1, int32x4x2_t *row1, int32x4x2_t *row2)
|
|
{
|
|
const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0);
|
|
const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0);
|
|
const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1);
|
|
const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1);
|
|
row1->val[0] = vmlal_n_s16(a2, vget_low_s16(b), c0);
|
|
row1->val[1] = vmlal_n_s16(a3, vget_high_s16(b), c0);
|
|
row2->val[0] = vmlsl_n_s16(a0, vget_low_s16(b), c1);
|
|
row2->val[1] = vmlsl_n_s16(a1, vget_high_s16(b), c1);
|
|
}
|
|
|
|
static INLINE void butterfly_one_coeff_32_32(
|
|
int32x4x2_t a, int32x4x2_t b, int32_t c, int32x4x2_t *row1, int32x4x2_t *row2)
|
|
{
|
|
const int32x4_t a0 = vmulq_n_s32(a.val[0], c);
|
|
const int32x4_t a1 = vmulq_n_s32(a.val[1], c);
|
|
row1->val[0] = vmlaq_n_s32(a0, b.val[0], c);
|
|
row1->val[1] = vmlaq_n_s32(a1, b.val[1], c);
|
|
row2->val[0] = vmlsq_n_s32(a0, b.val[0], c);
|
|
row2->val[1] = vmlsq_n_s32(a1, b.val[1], c);
|
|
}
|
|
|
|
static INLINE void butterfly_two_coeff_32_32(
|
|
int32x4x2_t a, int32x4x2_t b, int32_t c0, int32_t c1, int32x4x2_t *row1, int32x4x2_t *row2)
|
|
{
|
|
const int32x4_t a0 = vmulq_n_s32(a.val[0], c0);
|
|
const int32x4_t a1 = vmulq_n_s32(a.val[1], c0);
|
|
const int32x4_t a2 = vmulq_n_s32(a.val[0], c1);
|
|
const int32x4_t a3 = vmulq_n_s32(a.val[1], c1);
|
|
row1->val[0] = vmlaq_n_s32(a2, b.val[0], c0);
|
|
row1->val[1] = vmlaq_n_s32(a3, b.val[1], c0);
|
|
row2->val[0] = vmlsq_n_s32(a0, b.val[0], c1);
|
|
row2->val[1] = vmlsq_n_s32(a1, b.val[1], c1);
|
|
}
|
|
|
|
// Transpose 8x8 to a new location. Don't use transpose_neon.h because those
|
|
// are all in-place.
|
|
static INLINE void transpose_8x8(int32x4x2_t *a /*[8][2]*/, int32x4x2_t *b)
|
|
{
|
|
const int32x4x2_t c0 = vtrnq_s32(a[0].val[0], a[1].val[0]);
|
|
const int32x4x2_t c1 = vtrnq_s32(a[2].val[0], a[3].val[0]);
|
|
const int32x4x2_t c2 = vtrnq_s32(a[4].val[0], a[5].val[0]);
|
|
const int32x4x2_t c3 = vtrnq_s32(a[6].val[0], a[7].val[0]);
|
|
const int32x4x2_t c4 = vtrnq_s32(a[0].val[1], a[1].val[1]);
|
|
const int32x4x2_t c5 = vtrnq_s32(a[2].val[1], a[3].val[1]);
|
|
const int32x4x2_t c6 = vtrnq_s32(a[4].val[1], a[5].val[1]);
|
|
const int32x4x2_t c7 = vtrnq_s32(a[6].val[1], a[7].val[1]);
|
|
|
|
const int32x4x2_t d0 = vtrnq_s64_to_s32(c0.val[0], c1.val[0]);
|
|
const int32x4x2_t d1 = vtrnq_s64_to_s32(c0.val[1], c1.val[1]);
|
|
const int32x4x2_t d2 = vtrnq_s64_to_s32(c2.val[0], c3.val[0]);
|
|
const int32x4x2_t d3 = vtrnq_s64_to_s32(c2.val[1], c3.val[1]);
|
|
const int32x4x2_t d4 = vtrnq_s64_to_s32(c4.val[0], c5.val[0]);
|
|
const int32x4x2_t d5 = vtrnq_s64_to_s32(c4.val[1], c5.val[1]);
|
|
const int32x4x2_t d6 = vtrnq_s64_to_s32(c6.val[0], c7.val[0]);
|
|
const int32x4x2_t d7 = vtrnq_s64_to_s32(c6.val[1], c7.val[1]);
|
|
|
|
b[0].val[0] = d0.val[0];
|
|
b[0].val[1] = d2.val[0];
|
|
b[1].val[0] = d1.val[0];
|
|
b[1].val[1] = d3.val[0];
|
|
b[2].val[0] = d0.val[1];
|
|
b[2].val[1] = d2.val[1];
|
|
b[3].val[0] = d1.val[1];
|
|
b[3].val[1] = d3.val[1];
|
|
b[4].val[0] = d4.val[0];
|
|
b[4].val[1] = d6.val[0];
|
|
b[5].val[0] = d5.val[0];
|
|
b[5].val[1] = d7.val[0];
|
|
b[6].val[0] = d4.val[1];
|
|
b[6].val[1] = d6.val[1];
|
|
b[7].val[0] = d5.val[1];
|
|
b[7].val[1] = d7.val[1];
|
|
}
|
|
|
|
static void dct_body_16_32(int16x8_t *in /*[16]*/, int32x4x2_t *out /*[16]*/)
|
|
{
|
|
int16x8_t s[8];
|
|
int16x8_t x[4];
|
|
int32x4x2_t tmp0, tmp1, tmp2, tmp3;
|
|
int32x4x2_t tmp4, tmp5, tmp6, tmp7;
|
|
|
|
s[0] = vaddq_s16(in[0], in[7]);
|
|
s[1] = vaddq_s16(in[1], in[6]);
|
|
s[2] = vaddq_s16(in[2], in[5]);
|
|
s[3] = vaddq_s16(in[3], in[4]);
|
|
s[4] = vsubq_s16(in[3], in[4]);
|
|
s[5] = vsubq_s16(in[2], in[5]);
|
|
s[6] = vsubq_s16(in[1], in[6]);
|
|
s[7] = vsubq_s16(in[0], in[7]);
|
|
|
|
x[0] = vaddq_s16(s[0], s[3]);
|
|
x[1] = vaddq_s16(s[1], s[2]);
|
|
x[2] = vsubq_s16(s[1], s[2]);
|
|
x[3] = vsubq_s16(s[0], s[3]);
|
|
|
|
// Type 1
|
|
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
|
|
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
|
|
butterfly_one_coeff_16_32(x[0], x[1], 64, &out[0], &out[8]);
|
|
|
|
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
|
|
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
|
|
butterfly_two_coeff_16_32(x[3], x[2], 36, 83, &out[4], &out[12]);
|
|
|
|
// Type 2
|
|
butterfly_two_coeff_16_32(s[7], s[4], 18, 89, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(s[5], s[6], 75, 50, &tmp2, &tmp3);
|
|
|
|
out[2].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
|
|
out[2].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
|
|
|
|
out[14].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
|
|
out[14].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
|
|
|
|
butterfly_two_coeff_16_32(s[7], s[4], 75, 50, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(s[5], s[6], -89, 18, &tmp2, &tmp3);
|
|
|
|
out[10].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
|
|
out[10].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
|
|
|
|
out[6].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
|
|
out[6].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
|
|
|
|
// Type 3
|
|
butterfly_two_coeff_16_32(in[8], in[15], 9, -90, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(in[9], in[14], 87, 25, &tmp2, &tmp3);
|
|
butterfly_two_coeff_16_32(in[10], in[13], 43, -80, &tmp4, &tmp5);
|
|
butterfly_two_coeff_16_32(in[11], in[12], 70, 57, &tmp6, &tmp7);
|
|
|
|
out[1].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[1].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
out[15].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[15].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
butterfly_two_coeff_16_32(in[8], in[15], 87, -25, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(in[9], in[14], -70, -57, &tmp2, &tmp3);
|
|
butterfly_two_coeff_16_32(in[10], in[13], 9, -90, &tmp4, &tmp5);
|
|
butterfly_two_coeff_16_32(in[11], in[12], -80, 43, &tmp6, &tmp7);
|
|
|
|
out[3].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[3].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
out[13].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[13].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
butterfly_two_coeff_16_32(in[8], in[15], 43, -80, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(in[9], in[14], 9, 90, &tmp2, &tmp3);
|
|
butterfly_two_coeff_16_32(in[10], in[13], 57, 70, &tmp4, &tmp5);
|
|
butterfly_two_coeff_16_32(in[11], in[12], -87, -25, &tmp6, &tmp7);
|
|
|
|
out[5].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[5].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
out[11].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[11].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
butterfly_two_coeff_16_32(in[8], in[15], 70, -57, &tmp0, &tmp1);
|
|
butterfly_two_coeff_16_32(in[9], in[14], -80, 43, &tmp2, &tmp3);
|
|
butterfly_two_coeff_16_32(in[10], in[13], -87, 25, &tmp4, &tmp5);
|
|
butterfly_two_coeff_16_32(in[11], in[12], 90, -9, &tmp6, &tmp7);
|
|
|
|
out[7].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[7].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
out[9].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[9].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
}
|
|
|
|
static void dct_body_32_32(int32x4x2_t *in /*[16]*/, int32x4x2_t *out /*[16]*/)
|
|
{
|
|
int32x4x2_t s[8];
|
|
int32x4x2_t x[4];
|
|
int32x4x2_t tmp0, tmp1, tmp2, tmp3;
|
|
int32x4x2_t tmp4, tmp5, tmp6, tmp7;
|
|
WORD32 i;
|
|
|
|
for(i = 0; i < 2; i++)
|
|
{
|
|
s[0].val[i] = vaddq_s32(in[0].val[i], in[7].val[i]);
|
|
s[1].val[i] = vaddq_s32(in[1].val[i], in[6].val[i]);
|
|
s[2].val[i] = vaddq_s32(in[2].val[i], in[5].val[i]);
|
|
s[3].val[i] = vaddq_s32(in[3].val[i], in[4].val[i]);
|
|
s[4].val[i] = vsubq_s32(in[3].val[i], in[4].val[i]);
|
|
s[5].val[i] = vsubq_s32(in[2].val[i], in[5].val[i]);
|
|
s[6].val[i] = vsubq_s32(in[1].val[i], in[6].val[i]);
|
|
s[7].val[i] = vsubq_s32(in[0].val[i], in[7].val[i]);
|
|
|
|
x[0].val[i] = vaddq_s32(s[0].val[i], s[3].val[i]);
|
|
x[1].val[i] = vaddq_s32(s[1].val[i], s[2].val[i]);
|
|
x[2].val[i] = vsubq_s32(s[1].val[i], s[2].val[i]);
|
|
x[3].val[i] = vsubq_s32(s[0].val[i], s[3].val[i]);
|
|
}
|
|
|
|
// Type 1
|
|
// out[0] = fdct_round_shift((x0 + x1) * cospi_16_64)
|
|
// out[8] = fdct_round_shift((x0 - x1) * cospi_16_64)
|
|
butterfly_one_coeff_32_32(x[0], x[1], 64, &out[0], &out[8]);
|
|
// out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64);
|
|
// out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64);
|
|
butterfly_two_coeff_32_32(x[3], x[2], 36, 83, &out[4], &out[12]);
|
|
|
|
// Type 2
|
|
butterfly_two_coeff_32_32(s[7], s[4], 18, 89, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(s[5], s[6], 75, 50, &tmp2, &tmp3);
|
|
|
|
out[2].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
|
|
out[2].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
|
|
|
|
out[14].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
|
|
out[14].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
|
|
|
|
butterfly_two_coeff_32_32(s[7], s[4], 75, 50, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(s[5], s[6], -89, 18, &tmp2, &tmp3);
|
|
|
|
out[10].val[0] = vaddq_s32(tmp0.val[0], tmp2.val[0]);
|
|
out[10].val[1] = vaddq_s32(tmp0.val[1], tmp2.val[1]);
|
|
|
|
out[6].val[0] = vaddq_s32(tmp1.val[0], tmp3.val[0]);
|
|
out[6].val[1] = vaddq_s32(tmp1.val[1], tmp3.val[1]);
|
|
|
|
// Type 3
|
|
butterfly_two_coeff_32_32(in[8], in[15], 9, -90, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(in[9], in[14], 87, 25, &tmp2, &tmp3);
|
|
butterfly_two_coeff_32_32(in[10], in[13], 43, -80, &tmp4, &tmp5);
|
|
butterfly_two_coeff_32_32(in[11], in[12], 70, 57, &tmp6, &tmp7);
|
|
|
|
out[1].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[1].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
out[15].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[15].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
butterfly_two_coeff_32_32(in[8], in[15], 87, -25, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(in[9], in[14], -70, -57, &tmp2, &tmp3);
|
|
butterfly_two_coeff_32_32(in[10], in[13], 9, -90, &tmp4, &tmp5);
|
|
butterfly_two_coeff_32_32(in[11], in[12], -80, 43, &tmp6, &tmp7);
|
|
|
|
out[3].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[3].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
out[13].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[13].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
butterfly_two_coeff_32_32(in[8], in[15], 43, -80, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(in[9], in[14], 9, 90, &tmp2, &tmp3);
|
|
butterfly_two_coeff_32_32(in[10], in[13], 57, 70, &tmp4, &tmp5);
|
|
butterfly_two_coeff_32_32(in[11], in[12], -87, -25, &tmp6, &tmp7);
|
|
|
|
out[5].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[5].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
|
|
out[11].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[11].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
butterfly_two_coeff_32_32(in[8], in[15], 70, -57, &tmp0, &tmp1);
|
|
butterfly_two_coeff_32_32(in[9], in[14], -80, 43, &tmp2, &tmp3);
|
|
butterfly_two_coeff_32_32(in[10], in[13], -87, 25, &tmp4, &tmp5);
|
|
butterfly_two_coeff_32_32(in[11], in[12], 90, -9, &tmp6, &tmp7);
|
|
|
|
out[7].val[0] = add4(tmp0.val[0], tmp3.val[0], tmp4.val[0], tmp7.val[0]);
|
|
out[7].val[1] = add4(tmp0.val[1], tmp3.val[1], tmp4.val[1], tmp7.val[1]);
|
|
|
|
out[9].val[0] = add4(tmp1.val[0], tmp2.val[0], tmp5.val[0], tmp6.val[0]);
|
|
out[9].val[1] = add4(tmp1.val[1], tmp2.val[1], tmp5.val[1], tmp6.val[1]);
|
|
}
|
|
|
|
/**
|
|
*******************************************************************************
|
|
*
|
|
* @brief
|
|
* This function performs residue calculation and forward transform on
|
|
* input pixels
|
|
*
|
|
* @par Description:
|
|
* Performs residue calculation by subtracting source and prediction and
|
|
* followed by forward transform
|
|
*
|
|
* @param[in] pu1_src
|
|
* Input 16x16 pixels
|
|
*
|
|
* @param[in] pu1_pred
|
|
* Prediction data
|
|
*
|
|
* @param[in] pi2_tmp
|
|
* Temporary buffer of size 16x16
|
|
*
|
|
* @param[out] pi2_dst
|
|
* Output 16x16 coefficients
|
|
*
|
|
* @param[in] src_strd
|
|
* Input stride
|
|
*
|
|
* @param[in] pred_strd
|
|
* Prediction Stride
|
|
*
|
|
* @param[in] dst_strd
|
|
* Output Stride
|
|
*
|
|
* @param[in] e_chroma_plane
|
|
* Enum singalling chroma plane
|
|
*
|
|
* @returns Void
|
|
*
|
|
* @remarks
|
|
* None
|
|
*
|
|
*******************************************************************************
|
|
*/
|
|
UWORD32 ihevc_resi_trans_16x16_neon(
|
|
UWORD8 *pu1_src,
|
|
UWORD8 *pu1_pred,
|
|
WORD32 *pi4_temp,
|
|
WORD16 *pi2_dst,
|
|
WORD32 src_strd,
|
|
WORD32 pred_strd,
|
|
WORD32 dst_strd,
|
|
CHROMA_PLANE_ID_T e_chroma_plane)
|
|
{
|
|
UWORD32 u4_blk_sad = 0;
|
|
WORD32 chroma_flag;
|
|
uint8x8_t temp0[16], temp1[16];
|
|
int16x8_t temp2[16], temp3[16];
|
|
int32x4_t tmp_a, tmp_b;
|
|
int64x2_t tmp_c;
|
|
int32x2_t sad_v;
|
|
int32x4x2_t out0[16], out1[16], temp4[16], temp5[16];
|
|
|
|
(void)pi4_temp;
|
|
chroma_flag = e_chroma_plane != NULL_PLANE;
|
|
/* Residue + Forward Transform 1st stage */
|
|
// Left half.
|
|
load(pu1_src, src_strd, temp0, e_chroma_plane);
|
|
load(pu1_pred, pred_strd, temp1, e_chroma_plane);
|
|
|
|
tmp_a = diff(temp0, temp1, temp2);
|
|
cross_input_16(temp2, temp3);
|
|
dct_body_16_32(temp3, out0);
|
|
|
|
// Right half.
|
|
load(pu1_src + 8 * (1 + chroma_flag), src_strd, temp0, e_chroma_plane);
|
|
load(pu1_pred + 8 * (1 + chroma_flag), pred_strd, temp1, e_chroma_plane);
|
|
|
|
tmp_b = diff(temp0, temp1, temp2);
|
|
cross_input_16(temp2, temp3);
|
|
dct_body_16_32(temp3, out1);
|
|
|
|
tmp_a = vaddq_s32(tmp_a, tmp_b);
|
|
tmp_c = vpaddlq_s32(tmp_a);
|
|
sad_v = vadd_s32(vreinterpret_s32_s64(vget_low_s64(tmp_c)),
|
|
vreinterpret_s32_s64(vget_high_s64(tmp_c)));
|
|
u4_blk_sad = vget_lane_s32(sad_v, 0);
|
|
|
|
|
|
// Transpose top left and top right quarters into one contiguous location to
|
|
// process to the top half.
|
|
transpose_8x8(&out0[0], &temp4[0]);
|
|
transpose_8x8(&out1[0], &temp4[8]);
|
|
|
|
cross_input_32(temp4, temp5);
|
|
dct_body_32_32(temp5, temp4);
|
|
partial_round_shift(temp4, temp2);
|
|
transpose_s16_8x8(
|
|
&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]);
|
|
transpose_s16_8x8(
|
|
&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]);
|
|
|
|
store(pi2_dst, &temp2[0]);
|
|
store(pi2_dst + 8, &temp2[8]);
|
|
pi2_dst += 8 * dst_strd;
|
|
|
|
// Transpose bottom left and bottom right quarters into one contiguous
|
|
// location to process to the bottom half.
|
|
transpose_8x8(&out0[8], &out1[0]);
|
|
transpose_s32_8x8(
|
|
&out1[8], &out1[9], &out1[10], &out1[11], &out1[12], &out1[13], &out1[14], &out1[15]);
|
|
|
|
cross_input_32(out1, temp5);
|
|
dct_body_32_32(temp5, temp4);
|
|
partial_round_shift(temp4, temp2);
|
|
transpose_s16_8x8(
|
|
&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]);
|
|
transpose_s16_8x8(
|
|
&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]);
|
|
store(pi2_dst, &temp2[0]);
|
|
store(pi2_dst + 8, &temp2[8]);
|
|
|
|
return u4_blk_sad;
|
|
}
|