/****************************************************************************** * * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ***************************************************************************** * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore */ /** ******************************************************************************* * @file * ihevce_had_compute_neon.c * * @brief * Contains intrinsic definitions of functions for computing had * * @author * Ittiam * * @par List of Functions: * * @remarks * None * ******************************************************************************** */ /*****************************************************************************/ /* File Includes */ /*****************************************************************************/ /* System include files */ #include #include #include /* User include files */ #include "ihevc_typedefs.h" #include "itt_video_api.h" #include "ihevc_cmn_utils_neon.h" #include "ihevce_had_satd.h" #include "ihevce_cmn_utils_instr_set_router.h" /*****************************************************************************/ /* Globals */ /*****************************************************************************/ const int16_t gu2_dc_mask[8] = { 0x0000, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff }; /*****************************************************************************/ /* Function Macros */ /*****************************************************************************/ #define RESIDUE(k, is_chroma) \ if(!is_chroma) \ { \ const uint8x8_t s##k = vld1_u8(pu1_src); \ const uint8x8_t p##k = vld1_u8(pu1_pred); \ *r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \ pu1_src += src_strd; \ pu1_pred += pred_strd; \ } \ else \ { \ const uint8x8_t s##k = vld2_u8(pu1_src).val[0]; \ const uint8x8_t p##k = vld2_u8(pu1_pred).val[0]; \ *r##k = vreinterpretq_s16_u16(vsubl_u8(s##k, p##k)); \ pu1_src += src_strd; \ pu1_pred += pred_strd; \ } /*****************************************************************************/ /* Function Definitions */ /*****************************************************************************/ static INLINE void hadamard4x4_2_one_pass(int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3) { const int16x8_t a0 = vaddq_s16(*r0, *r2); const int16x8_t a1 = vaddq_s16(*r1, *r3); const int16x8_t a2 = vsubq_s16(*r0, *r2); const int16x8_t a3 = vsubq_s16(*r1, *r3); *r0 = vaddq_s16(a0, a1); *r1 = vsubq_s16(a0, a1); *r2 = vaddq_s16(a2, a3); *r3 = vsubq_s16(a2, a3); } static INLINE void hadamard4x4_2( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3) { // compute error between src and pred RESIDUE(0, 0); RESIDUE(1, 0); RESIDUE(2, 0); RESIDUE(3, 0); // vertical hadamard tx hadamard4x4_2_one_pass(r0, r1, r2, r3); // transpose transpose_s16_4x4q(r0, r1, r2, r3); // horizontal hadamard tx hadamard4x4_2_one_pass(r0, r1, r2, r3); } static INLINE void hadamard4x4_4( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3, int16x8_t *r4, int16x8_t *r5, int16x8_t *r6, int16x8_t *r7) { // hadamard 4x4_2n hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r0, r1, r2, r3); // hadamard 4x4_2n pu1_src += (4 * src_strd); pu1_pred += (4 * pred_strd); hadamard4x4_2(pu1_src, src_strd, pu1_pred, pred_strd, r4, r5, r6, r7); } static INLINE WORD32 hadamard_sad4x4_4(int16x8_t *a, WORD32 *pi4_hsad, WORD32 hsad_stride) { int16x8_t p[8]; int32x4_t b01, b23; int64x2_t c01, c23; int32x2_t d01, d23; // satd p[0] = vabsq_s16(a[0]); p[1] = vabsq_s16(a[1]); p[0] = vaddq_s16(p[0], p[1]); p[2] = vabsq_s16(a[2]); p[3] = vabsq_s16(a[3]); p[2] = vaddq_s16(p[2], p[3]); p[4] = vabsq_s16(a[4]); p[5] = vabsq_s16(a[5]); p[4] = vaddq_s16(p[4], p[5]); p[6] = vabsq_s16(a[6]); p[7] = vabsq_s16(a[7]); p[6] = vaddq_s16(p[6], p[7]); p[0] = vaddq_s16(p[0], p[2]); b01 = vpaddlq_s16(p[0]); c01 = vpaddlq_s32(b01); d01 = vrshrn_n_s64(c01, 2); vst1_s32(pi4_hsad, d01); pi4_hsad += hsad_stride; p[4] = vaddq_s16(p[4], p[6]); b23 = vpaddlq_s16(p[4]); c23 = vpaddlq_s32(b23); d23 = vrshrn_n_s64(c23, 2); vst1_s32(pi4_hsad, d23); d01 = vadd_s32(d01, d23); return (WORD32)(vget_lane_s64(vpaddl_s32(d01), 0)); } static INLINE WORD32 hadamard_sad8x8_using4x4(int16x8_t *a, WORD32 *early_cbf, WORD32 i4_frm_qstep) { int16x8_t p[8]; const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); int32x4_t b; int64x2_t c; int64_t satd; WORD32 i; for(i = 0; i < 4; i++) { int16x8_t p0 = vaddq_s16(a[i], a[i + 4]); int16x8_t p1 = vsubq_s16(a[i], a[i + 4]); int16x4_t q0 = vadd_s16(vget_low_s16(p0), vget_high_s16(p0)); int16x4_t q1 = vsub_s16(vget_low_s16(p0), vget_high_s16(p0)); int16x4_t q2 = vadd_s16(vget_low_s16(p1), vget_high_s16(p1)); int16x4_t q3 = vsub_s16(vget_low_s16(p1), vget_high_s16(p1)); a[i] = vcombine_s16(q0, q2); a[i + 4] = vcombine_s16(q1, q3); } #define EARLY_EXIT(k) \ { \ p[k] = vabsq_s16(a[k]); \ if(*early_cbf == 0) \ { \ uint16x8_t cmp; \ cmp = vcgtq_s16(p[k], threshold); \ if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \ vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \ { \ *early_cbf = 1; \ } \ } \ } // satd EARLY_EXIT(0); EARLY_EXIT(1); p[0] = vaddq_s16(p[0], p[1]); EARLY_EXIT(2); EARLY_EXIT(3); p[2] = vaddq_s16(p[2], p[3]); EARLY_EXIT(4); EARLY_EXIT(5); p[4] = vaddq_s16(p[4], p[5]); EARLY_EXIT(6); EARLY_EXIT(7); #undef EARLY_EXIT p[6] = vaddq_s16(p[6], p[7]); p[0] = vaddq_s16(p[0], p[2]); p[4] = vaddq_s16(p[4], p[6]); p[0] = vaddq_s16(p[0], p[4]); b = vpaddlq_s16(p[0]); c = vpaddlq_s32(b); satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0); return ((satd + 4) >> 3); } static INLINE void hadamard8x8_one_pass( int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3, int16x8_t *r4, int16x8_t *r5, int16x8_t *r6, int16x8_t *r7) { const int16x8_t a0 = vaddq_s16(*r0, *r4); const int16x8_t a4 = vsubq_s16(*r0, *r4); const int16x8_t a1 = vaddq_s16(*r1, *r5); const int16x8_t a5 = vsubq_s16(*r1, *r5); const int16x8_t a2 = vaddq_s16(*r2, *r6); const int16x8_t a6 = vsubq_s16(*r2, *r6); const int16x8_t a3 = vaddq_s16(*r3, *r7); const int16x8_t a7 = vsubq_s16(*r3, *r7); const int16x8_t b0 = vaddq_s16(a0, a2); const int16x8_t b2 = vsubq_s16(a0, a2); const int16x8_t b1 = vaddq_s16(a1, a3); const int16x8_t b3 = vsubq_s16(a1, a3); const int16x8_t b4 = vaddq_s16(a4, a6); const int16x8_t b6 = vsubq_s16(a4, a6); const int16x8_t b5 = vaddq_s16(a5, a7); const int16x8_t b7 = vsubq_s16(a5, a7); *r0 = vaddq_s16(b0, b1); *r1 = vsubq_s16(b0, b1); *r2 = vaddq_s16(b2, b3); *r3 = vsubq_s16(b2, b3); *r4 = vaddq_s16(b4, b5); *r5 = vsubq_s16(b4, b5); *r6 = vaddq_s16(b6, b7); *r7 = vsubq_s16(b6, b7); } static INLINE void hadamard8x8( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, int16x8_t *r0, int16x8_t *r1, int16x8_t *r2, int16x8_t *r3, int16x8_t *r4, int16x8_t *r5, int16x8_t *r6, int16x8_t *r7, WORD32 is_chroma) { // compute error between src and pred RESIDUE(0, is_chroma); RESIDUE(1, is_chroma); RESIDUE(2, is_chroma); RESIDUE(3, is_chroma); RESIDUE(4, is_chroma); RESIDUE(5, is_chroma); RESIDUE(6, is_chroma); RESIDUE(7, is_chroma); // vertical hadamard tx hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7); // transpose transpose_s16_8x8(r0, r1, r2, r3, r4, r5, r6, r7); // horizontal hadamard tx hadamard8x8_one_pass(r0, r1, r2, r3, r4, r5, r6, r7); } static INLINE UWORD32 ihevce_HAD_8x8_8bit_plane_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD32 is_chroma, WORD32 ac_only) { int16x8_t a0, a1, a2, a3, a4, a5, a6, a7; int32x4_t b; int64x2_t c; int64_t satd; // hadamard 8x8 hadamard8x8( pu1_src, src_strd, pu1_pred, pred_strd, &a0, &a1, &a2, &a3, &a4, &a5, &a6, &a7, is_chroma); if(ac_only) { const int16x8_t mask = vld1q_s16(gu2_dc_mask); a0 = vandq_s16(a0, mask); } // satd a0 = vabsq_s16(a0); a1 = vabsq_s16(a1); a0 = vaddq_s16(a0, a1); a2 = vabsq_s16(a2); a3 = vabsq_s16(a3); a2 = vaddq_s16(a2, a3); a4 = vabsq_s16(a4); a5 = vabsq_s16(a5); a4 = vaddq_s16(a4, a5); a6 = vabsq_s16(a6); a7 = vabsq_s16(a7); a6 = vaddq_s16(a6, a7); a0 = vaddq_s16(a0, a2); a4 = vaddq_s16(a4, a6); a0 = vaddq_s16(a0, a4); b = vpaddlq_s16(a0); c = vpaddlq_s32(b); satd = vget_lane_s64(vadd_s64(vget_low_s64(c), vget_high_s64(c)), 0); return ((satd + 4) >> 3); } static INLINE UWORD32 ihevce_HAD_4x4_8bit_plane_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD32 is_chroma, WORD32 ac_only) { uint8x16_t src_u8, pred_u8; int16x8_t res_01, res_23; int16x4_t h[4]; int16x4_t v[4]; int16x4x2_t trans_4[2]; int16x8_t combined_rows[4]; int32x4x2_t trans_8; int32x4_t sad_32_4[3]; int32x2_t sad_32_2; int64x1_t sad_64_1; int32_t sad; if(!is_chroma) { src_u8 = load_unaligned_u8q(pu1_src, src_strd); pred_u8 = load_unaligned_u8q(pu1_pred, pred_strd); } else { src_u8 = load_unaligned_u8qi(pu1_src, src_strd); pred_u8 = load_unaligned_u8qi(pu1_pred, pred_strd); } res_01 = vreinterpretq_s16_u16(vsubl_u8(vget_low_u8(src_u8), vget_low_u8(pred_u8))); res_23 = vreinterpretq_s16_u16(vsubl_u8(vget_high_u8(src_u8), vget_high_u8(pred_u8))); h[0] = vadd_s16(vget_low_s16(res_01), vget_high_s16(res_23)); h[1] = vadd_s16(vget_high_s16(res_01), vget_low_s16(res_23)); h[2] = vsub_s16(vget_high_s16(res_01), vget_low_s16(res_23)); h[3] = vsub_s16(vget_low_s16(res_01), vget_high_s16(res_23)); v[0] = vadd_s16(h[0], h[1]); v[1] = vadd_s16(h[3], h[2]); v[2] = vsub_s16(h[0], h[1]); v[3] = vsub_s16(h[3], h[2]); trans_4[0] = vtrn_s16(v[0], v[2]); trans_4[1] = vtrn_s16(v[1], v[3]); combined_rows[0] = vcombine_s16(trans_4[0].val[0], trans_4[1].val[0]); combined_rows[1] = vcombine_s16(trans_4[0].val[1], trans_4[1].val[1]); combined_rows[2] = vaddq_s16(combined_rows[0], combined_rows[1]); combined_rows[3] = vsubq_s16(combined_rows[0], combined_rows[1]); trans_8 = vtrnq_s32(vreinterpretq_s32_s16(combined_rows[2]), vreinterpretq_s32_s16(combined_rows[3])); combined_rows[0] = vaddq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1])); combined_rows[0] = vabsq_s16(combined_rows[0]); combined_rows[1] = vsubq_s16(vreinterpretq_s16_s32(trans_8.val[0]), vreinterpretq_s16_s32(trans_8.val[1])); combined_rows[1] = vabsq_s16(combined_rows[1]); if(ac_only) { const int16x8_t mask = vld1q_s16(gu2_dc_mask); combined_rows[0] = vandq_s16(combined_rows[0], mask); } sad_32_4[0] = vpaddlq_s16(combined_rows[0]); sad_32_4[1] = vpaddlq_s16(combined_rows[1]); sad_32_4[2] = vaddq_s32(sad_32_4[0], sad_32_4[1]); sad_32_2 = vadd_s32(vget_high_s32(sad_32_4[2]), vget_low_s32(sad_32_4[2])); sad_64_1 = vpaddl_s32(sad_32_2); sad = vget_lane_s64(sad_64_1, 0); return ((sad + 2) >> 2); } UWORD32 ihevce_HAD_4x4_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0); } UWORD32 ihevce_chroma_compute_AC_HAD_4x4_8bit_neon( UWORD8 *pu1_origin, WORD32 src_strd, UWORD8 *pu1_pred_buf, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_4x4_8bit_plane_neon(pu1_origin, src_strd, pu1_pred_buf, pred_strd, 1, 1); } UWORD32 ihevce_HAD_8x8_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 0); } UWORD32 ihevce_compute_ac_had_8x8_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 0, 1); } UWORD32 ihevce_HAD_16x16_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { int16x8_t b0[8]; int16x8_t b1[8]; int16x8_t b2[8]; int16x8_t b3[8]; uint32x4_t sum = vdupq_n_u32(0); uint64x2_t c; uint64_t satd; WORD32 i; (void)pi2_dst; (void)dst_strd; // hadamard 8x8 - b0 hadamard8x8( pu1_src, src_strd, pu1_pred, pred_strd, &b0[0], &b0[1], &b0[2], &b0[3], &b0[4], &b0[5], &b0[6], &b0[7], 0); // hadamard 8x8 - b1 hadamard8x8( pu1_src + 8, src_strd, pu1_pred + 8, pred_strd, &b1[0], &b1[1], &b1[2], &b1[3], &b1[4], &b1[5], &b1[6], &b1[7], 0); // hadamard 8x8 - b2 hadamard8x8( pu1_src + (8 * src_strd), src_strd, pu1_pred + (8 * pred_strd), pred_strd, &b2[0], &b2[1], &b2[2], &b2[3], &b2[4], &b2[5], &b2[6], &b2[7], 0); // hadamard 8x8 - b3 hadamard8x8( pu1_src + (8 * src_strd) + 8, src_strd, pu1_pred + (8 * pred_strd) + 8, pred_strd, &b3[0], &b3[1], &b3[2], &b3[3], &b3[4], &b3[5], &b3[6], &b3[7], 0); for(i = 0; i < 8; i++) { int16x8_t p0 = vhaddq_s16(b0[i], b1[i]); int16x8_t p1 = vhsubq_s16(b0[i], b1[i]); int16x8_t p2 = vhaddq_s16(b2[i], b3[i]); int16x8_t p3 = vhsubq_s16(b2[i], b3[i]); int16x8_t q0 = vaddq_s16(p0, p2); int16x8_t q1 = vsubq_s16(p0, p2); int16x8_t q2 = vaddq_s16(p1, p3); int16x8_t q3 = vsubq_s16(p1, p3); uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1))); uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3))); uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); sum = vaddq_u32(sum, s0); sum = vaddq_u32(sum, s1); } c = vpaddlq_u32(sum); satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); return ((satd + 4) >> 3); } UWORD32 ihevce_chroma_HAD_4x4_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_4x4_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); } UWORD32 ihevce_chroma_HAD_8x8_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { (void)pi2_dst; (void)dst_strd; return ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); } UWORD32 ihevce_chroma_HAD_16x16_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { UWORD32 au4_satd[4]; (void)pi2_dst; (void)dst_strd; au4_satd[0] = ihevce_HAD_8x8_8bit_plane_neon(pu1_src, src_strd, pu1_pred, pred_strd, 1, 0); au4_satd[1] = ihevce_HAD_8x8_8bit_plane_neon(pu1_src + 16, src_strd, pu1_pred + 16, pred_strd, 1, 0); au4_satd[2] = ihevce_HAD_8x8_8bit_plane_neon( pu1_src + 8 * src_strd, src_strd, pu1_pred + 8 * pred_strd, pred_strd, 1, 0); au4_satd[3] = ihevce_HAD_8x8_8bit_plane_neon( pu1_src + 8 * src_strd + 16, src_strd, pu1_pred + 8 * pred_strd + 16, pred_strd, 1, 0); return au4_satd[0] + au4_satd[1] + au4_satd[2] + au4_satd[3]; } UWORD32 ihevce_HAD_32x32_8bit_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd) { int16x8_t a[4][4][8]; uint32x4_t sum = vdupq_n_u32(0); WORD32 b8, b16; uint64x2_t c; uint64_t satd; WORD32 i, j; (void)pi2_dst; (void)dst_strd; // hadamard 32x32 for(b16 = 0; b16 < 4; b16++) { UWORD8 *pu1_src_b16 = pu1_src + (b16 >> 1) * (src_strd * 16) + ((b16 & 1) * 16); UWORD8 *pu1_pred_b16 = pu1_pred + (b16 >> 1) * (pred_strd * 16) + ((b16 & 1) * 16); // hadamard 16x16 for(b8 = 0; b8 < 4; b8++) { UWORD8 *pu1_src_b8 = pu1_src_b16 + (b8 >> 1) * (src_strd * 8) + ((b8 & 1) * 8); UWORD8 *pu1_pred_b8 = pu1_pred_b16 + (b8 >> 1) * (pred_strd * 8) + ((b8 & 1) * 8); // hadamard 8x8 hadamard8x8( pu1_src_b8, src_strd, pu1_pred_b8, pred_strd, &a[b16][b8][0], &a[b16][b8][1], &a[b16][b8][2], &a[b16][b8][3], &a[b16][b8][4], &a[b16][b8][5], &a[b16][b8][6], &a[b16][b8][7], 0); } for(i = 0; i < 8; i++) { int16x8_t p0 = vhaddq_s16(a[b16][0][i], a[b16][1][i]); int16x8_t p1 = vhsubq_s16(a[b16][0][i], a[b16][1][i]); int16x8_t p2 = vhaddq_s16(a[b16][2][i], a[b16][3][i]); int16x8_t p3 = vhsubq_s16(a[b16][2][i], a[b16][3][i]); a[b16][0][i] = vaddq_s16(p0, p2); a[b16][1][i] = vsubq_s16(p0, p2); a[b16][2][i] = vaddq_s16(p1, p3); a[b16][3][i] = vsubq_s16(p1, p3); a[b16][0][i] = vshrq_n_s16(a[b16][0][i], 2); a[b16][1][i] = vshrq_n_s16(a[b16][1][i], 2); a[b16][2][i] = vshrq_n_s16(a[b16][2][i], 2); a[b16][3][i] = vshrq_n_s16(a[b16][3][i], 2); } } for(j = 0; j < 4; j++) { for(i = 0; i < 8; i++) { int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]); int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]); int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]); int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]); int16x8_t q0 = vaddq_s16(p0, p2); int16x8_t q1 = vsubq_s16(p0, p2); int16x8_t q2 = vaddq_s16(p1, p3); int16x8_t q3 = vsubq_s16(p1, p3); uint16x8_t r0 = vaddq_u16( vreinterpretq_u16_s16(vabsq_s16(q0)), vreinterpretq_u16_s16(vabsq_s16(q1))); uint16x8_t r1 = vaddq_u16( vreinterpretq_u16_s16(vabsq_s16(q2)), vreinterpretq_u16_s16(vabsq_s16(q3))); uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); sum = vaddq_u32(sum, s0); sum = vaddq_u32(sum, s1); } } c = vpaddlq_u32(sum); satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); return ((satd + 2) >> 2); } WORD32 ihevce_had4_4x4_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst4x4, WORD32 dst_strd, WORD32 *pi4_hsad, WORD32 hsad_stride, WORD32 i4_frm_qstep) { int16x8_t a[8]; (void)pi2_dst4x4; (void)dst_strd; (void)i4_frm_qstep; /* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */ hadamard4x4_4( pu1_src, src_strd, pu1_pred, pred_strd, &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); return hadamard_sad4x4_4(a, pi4_hsad, hsad_stride); } WORD32 ihevce_had_8x8_using_4_4x4_r_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd, WORD32 **ppi4_hsad, WORD32 **ppi4_tu_split, WORD32 **ppi4_tu_early_cbf, WORD32 pos_x_y_4x4, WORD32 num_4x4_in_row, WORD32 lambda, WORD32 lambda_q_shift, WORD32 i4_frm_qstep, WORD32 i4_cur_depth, WORD32 i4_max_depth, WORD32 i4_max_tr_size, WORD32 *pi4_tu_split_cost, void *pv_func_sel) { WORD32 pos_x = pos_x_y_4x4 & 0xFFFF; WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF; WORD32 *pi4_4x4_hsad; WORD32 *pi4_8x8_hsad; WORD32 *pi4_8x8_tu_split; WORD32 *pi4_8x8_tu_early_cbf; WORD32 cost_child, cost_parent; WORD32 best_cost; WORD32 early_cbf = 0; const UWORD8 u1_cur_tr_size = 8; WORD32 i; int16x8_t a[8]; (void)pv_func_sel; assert(pos_x >= 0); assert(pos_y >= 0); /* Initialize pointers to store 4x4 and 8x8 HAD SATDs */ pi4_4x4_hsad = ppi4_hsad[HAD_4x4] + pos_x + pos_y * num_4x4_in_row; pi4_8x8_hsad = ppi4_hsad[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); pi4_8x8_tu_split = ppi4_tu_split[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); pi4_8x8_tu_early_cbf = ppi4_tu_early_cbf[HAD_8x8] + (pos_x >> 1) + (pos_y >> 1) * (num_4x4_in_row >> 1); /* -------- Compute four 4x4 HAD Transforms of 8x8 in one call--------- */ hadamard4x4_4( pu1_src, src_strd, pu1_pred, pred_strd, &a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6], &a[7]); /* -------- cost child -------- */ cost_child = hadamard_sad4x4_4(a, pi4_4x4_hsad, num_4x4_in_row); /* 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */ cost_child += ((4) * lambda) >> (lambda_q_shift + 1); /* -------- cost parent -------- */ cost_parent = hadamard_sad8x8_using4x4(a, &early_cbf, i4_frm_qstep); for(i = 0; i < 8; i++, pi2_dst += dst_strd) vst1q_s16(pi2_dst, a[i]); if(i4_cur_depth < i4_max_depth) { if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size)) { *pi4_tu_split_cost += (4 * lambda) >> (lambda_q_shift + 1); best_cost = cost_child; best_cost <<= 1; best_cost++; pi4_8x8_tu_split[0] = 1; pi4_8x8_hsad[0] = cost_child; } else { best_cost = cost_parent; best_cost <<= 1; pi4_8x8_tu_split[0] = 0; pi4_8x8_hsad[0] = cost_parent; } } else { best_cost = cost_parent; best_cost <<= 1; pi4_8x8_tu_split[0] = 0; pi4_8x8_hsad[0] = cost_parent; } pi4_8x8_tu_early_cbf[0] = early_cbf; /* best cost has tu_split_flag at LSB(Least significant bit) */ return ((best_cost << 1) + early_cbf); } static WORD32 ihevce_compute_16x16HAD_using_8x8_neon( WORD16 *pi2_8x8_had, WORD32 had8_strd, WORD16 *pi2_dst, WORD32 dst_strd, WORD32 i4_frm_qstep, WORD32 *pi4_cbf) { int16x8_t b0[8]; int16x8_t b1[8]; int16x8_t b2[8]; int16x8_t b3[8]; const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); uint32x4_t sum = vdupq_n_u32(0); uint64x2_t c; uint64_t satd; WORD32 i; for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd) { b0[i] = vld1q_s16(pi2_8x8_had); b1[i] = vld1q_s16(pi2_8x8_had + 8); } for(i = 0; i < 8; i++, pi2_8x8_had += had8_strd) { b2[i] = vld1q_s16(pi2_8x8_had); b3[i] = vld1q_s16(pi2_8x8_had + 8); } #define EARLY_EXIT(k) \ { \ p##k = vabsq_s16(q##k); \ if(*pi4_cbf == 0) \ { \ uint16x8_t cmp; \ cmp = vcgtq_s16(p##k, threshold); \ if(vget_lane_s64(vreinterpret_s64_u16(vget_low_u16(cmp)), 0) || \ vget_lane_s64(vreinterpret_s64_u16(vget_high_u16(cmp)), 0)) \ { \ *pi4_cbf = 1; \ } \ } \ } for(i = 0; i < 8; i++, pi2_dst += dst_strd) { int16x8_t p0 = vhaddq_s16(b0[i], b1[i]); int16x8_t p1 = vhsubq_s16(b0[i], b1[i]); int16x8_t p2 = vhaddq_s16(b2[i], b3[i]); int16x8_t p3 = vhsubq_s16(b2[i], b3[i]); int16x8_t q0 = vaddq_s16(p0, p2); int16x8_t q1 = vsubq_s16(p0, p2); int16x8_t q2 = vaddq_s16(p1, p3); int16x8_t q3 = vsubq_s16(p1, p3); vst1q_s16(pi2_dst, q0); EARLY_EXIT(0); vst1q_s16(pi2_dst + 8, q1); EARLY_EXIT(1); vst1q_s16(pi2_dst + 8 * dst_strd, q2); EARLY_EXIT(2); vst1q_s16(pi2_dst + 8 * dst_strd + 8, q3); EARLY_EXIT(3); uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1)); uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3)); uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); sum = vaddq_u32(sum, s0); sum = vaddq_u32(sum, s1); } c = vpaddlq_u32(sum); satd = vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); return ((satd + 4) >> 3); } WORD32 ihevce_had_16x16_r_neon( UWORD8 *pu1_src, WORD32 src_strd, UWORD8 *pu1_pred, WORD32 pred_strd, WORD16 *pi2_dst, WORD32 dst_strd, WORD32 **ppi4_hsad, WORD32 **ppi4_tu_split, WORD32 **ppi4_tu_early_cbf, WORD32 pos_x_y_4x4, WORD32 num_4x4_in_row, WORD32 lambda, WORD32 lambda_q_shift, WORD32 i4_frm_qstep, WORD32 i4_cur_depth, WORD32 i4_max_depth, WORD32 i4_max_tr_size, WORD32 *pi4_tu_split_cost, void *pv_func_sel) { WORD16 ai2_8x8_had[256]; WORD32 *pi4_16x16_hsad; WORD32 *pi4_16x16_tu_split; WORD32 *pi4_16x16_tu_early_cbf; WORD32 best_cost, best_cost_tu_split; WORD32 tu_split_flag = 0; WORD32 i4_early_cbf_flag = 0, early_cbf = 0; WORD32 cost_parent, cost_child = 0; const UWORD8 u1_cur_tr_size = 16; WORD32 i; WORD16 *pi2_y0; UWORD8 *src, *pred; WORD32 pos_x_y_4x4_0; WORD32 pos_x = pos_x_y_4x4 & 0xFFFF; WORD32 pos_y = (pos_x_y_4x4 >> 16) & 0xFFFF; assert(pos_x >= 0); assert(pos_y >= 0); /* Initialize pointers to store 16x16 SATDs */ pi4_16x16_hsad = ppi4_hsad[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); pi4_16x16_tu_split = ppi4_tu_split[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); pi4_16x16_tu_early_cbf = ppi4_tu_early_cbf[HAD_16x16] + (pos_x >> 2) + (pos_y >> 2) * (num_4x4_in_row >> 2); /* -------- Compute four 8x8 HAD Transforms of 16x16 call--------- */ for(i = 0; i < 4; i++) { src = pu1_src + (i & 0x01) * 8 + (i >> 1) * src_strd * 8; pred = pu1_pred + (i & 0x01) * 8 + (i >> 1) * pred_strd * 8; pi2_y0 = ai2_8x8_had + (i & 0x01) * 8 + (i >> 1) * 16 * 8; pos_x_y_4x4_0 = pos_x_y_4x4 + (i & 0x01) * 2 + (i >> 1) * (2 << 16); best_cost_tu_split = ihevce_had_8x8_using_4_4x4_r_neon( src, src_strd, pred, pred_strd, pi2_y0, 16, ppi4_hsad, ppi4_tu_split, ppi4_tu_early_cbf, pos_x_y_4x4_0, num_4x4_in_row, lambda, lambda_q_shift, i4_frm_qstep, i4_cur_depth + 1, i4_max_depth, i4_max_tr_size, pi4_tu_split_cost, pv_func_sel); /* Cost is shifted by two bits for Tu_split_flag and early cbf flag */ best_cost = (best_cost_tu_split >> 2); /* Last but one bit stores the information regarding the TU_Split */ tu_split_flag += (best_cost_tu_split & 0x3) >> 1; /* Last bit stores the information regarding the early_cbf */ i4_early_cbf_flag += (best_cost_tu_split & 0x1); cost_child += best_cost; tu_split_flag <<= 1; i4_early_cbf_flag <<= 1; } /* -------- Compute 16x16 HAD Transform using 8x8 results ------------- */ pi2_y0 = ai2_8x8_had; /* Threshold currently passed as "0" */ cost_parent = ihevce_compute_16x16HAD_using_8x8_neon( pi2_y0, 16, pi2_dst, dst_strd, i4_frm_qstep, &early_cbf); /* 4 TU_Split flags , 4 CBF Flags, extra 1 becoz of the 0.5 bits per bin is assumed */ cost_child += ((4 + 4) * lambda) >> (lambda_q_shift + 1); i4_early_cbf_flag += early_cbf; /* Right now the depth is hard-coded to 4: The depth can be modified from the config file which decides the extent to which TU_REC needs to be done */ if(i4_cur_depth < i4_max_depth) { if((cost_child < cost_parent) || (i4_max_tr_size < u1_cur_tr_size)) { *pi4_tu_split_cost += ((4 + 4) * lambda) >> (lambda_q_shift + 1); tu_split_flag += 1; best_cost = cost_child; } else { tu_split_flag += 0; best_cost = cost_parent; } } else { tu_split_flag += 0; best_cost = cost_parent; } pi4_16x16_hsad[0] = best_cost; pi4_16x16_tu_split[0] = tu_split_flag; pi4_16x16_tu_early_cbf[0] = i4_early_cbf_flag; /*returning two values(best cost & tu_split_flag) as a single value*/ return ((best_cost << 10) + (tu_split_flag << 5) + i4_early_cbf_flag); } UWORD32 ihevce_compute_32x32HAD_using_16x16_neon( WORD16 *pi2_16x16_had, WORD32 had16_strd, WORD16 *pi2_dst, WORD32 dst_strd, WORD32 i4_frm_qstep, WORD32 *pi4_cbf) { int16x8_t a[4][4][8]; uint32x4_t sum = vdupq_n_u32(0); const int16x8_t threshold = vdupq_n_s16((int16_t)(i4_frm_qstep >> 8)); WORD32 b8, b16; uint64x2_t c; WORD32 i, j; (void)pi2_dst; (void)dst_strd; for(b16 = 0; b16 < 4; b16++) { WORD16 *pi2_b16 = pi2_16x16_had + (b16 >> 1) * (had16_strd * 16) + ((b16 & 1) * 16); for(b8 = 0; b8 < 4; b8++) { WORD16 *pi2_b8 = pi2_b16 + (b8 >> 1) * (had16_strd * 8) + ((b8 & 1) * 8); for(i = 0; i < 8; i++, pi2_b8 += had16_strd) { a[b16][b8][i] = vld1q_s16(pi2_b8); a[b16][b8][i] = vshrq_n_s16(a[b16][b8][i], 2); } } } for(j = 0; j < 4; j++) { for(i = 0; i < 8; i++) { int16x8_t p0 = vaddq_s16(a[0][j][i], a[1][j][i]); int16x8_t p1 = vsubq_s16(a[0][j][i], a[1][j][i]); int16x8_t p2 = vaddq_s16(a[2][j][i], a[3][j][i]); int16x8_t p3 = vsubq_s16(a[2][j][i], a[3][j][i]); int16x8_t q0 = vaddq_s16(p0, p2); int16x8_t q1 = vsubq_s16(p0, p2); int16x8_t q2 = vaddq_s16(p1, p3); int16x8_t q3 = vsubq_s16(p1, p3); EARLY_EXIT(0); EARLY_EXIT(1); EARLY_EXIT(2); EARLY_EXIT(3); uint16x8_t r0 = vaddq_u16(vreinterpretq_u16_s16(p0), vreinterpretq_u16_s16(p1)); uint16x8_t r1 = vaddq_u16(vreinterpretq_u16_s16(p2), vreinterpretq_u16_s16(p3)); uint32x4_t s0 = vaddl_u16(vget_low_u16(r0), vget_high_u16(r0)); uint32x4_t s1 = vaddl_u16(vget_low_u16(r1), vget_high_u16(r1)); sum = vaddq_u32(sum, s0); sum = vaddq_u32(sum, s1); } } c = vpaddlq_u32(sum); return vget_lane_u64(vadd_u64(vget_low_u64(c), vget_high_u64(c)), 0); }