Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
       3             :  *
       4             :  * This source code is subject to the terms of the BSD 2 Clause License and
       5             :  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
       6             :  * was not distributed with this source code in the LICENSE file, you can
       7             :  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
       8             :  * Media Patent License 1.0 was not distributed with this source code in the
       9             :  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
      10             :  */
      11             : 
      12             : #ifndef _HIGHBD_TXFM_UTILITY_SSE4_H
      13             : #define _HIGHBD_TXFM_UTILITY_SSE4_H
      14             : 
      15             : #include <smmintrin.h> /* SSE4.1 */
      16             : 
      17             : #define TRANSPOSE_4X4(x0, x1, x2, x3, y0, y1, y2, y3) \
      18             :   do {                                                \
      19             :     __m128i u0, u1, u2, u3;                           \
      20             :     u0 = _mm_unpacklo_epi32(x0, x1);                  \
      21             :     u1 = _mm_unpackhi_epi32(x0, x1);                  \
      22             :     u2 = _mm_unpacklo_epi32(x2, x3);                  \
      23             :     u3 = _mm_unpackhi_epi32(x2, x3);                  \
      24             :     y0 = _mm_unpacklo_epi64(u0, u2);                  \
      25             :     y1 = _mm_unpackhi_epi64(u0, u2);                  \
      26             :     y2 = _mm_unpacklo_epi64(u1, u3);                  \
      27             :     y3 = _mm_unpackhi_epi64(u1, u3);                  \
      28             :   } while (0)
      29             : 
      30             : static INLINE void transpose_4x4(const __m128i *in, __m128i *out) {
      31             :     TRANSPOSE_4X4(in[0], in[1], in[2], in[3], out[0], out[1], out[2], out[3]);
      32             : }
      33             : 
      34           0 : static INLINE void transpose_8x8(const __m128i *in, __m128i *out) {
      35           0 :     TRANSPOSE_4X4(in[0], in[2], in[4], in[6], out[0], out[2], out[4], out[6]);
      36           0 :     TRANSPOSE_4X4(in[1], in[3], in[5], in[7], out[8], out[10], out[12], out[14]);
      37           0 :     TRANSPOSE_4X4(in[8], in[10], in[12], in[14], out[1], out[3], out[5], out[7]);
      38           0 :     TRANSPOSE_4X4(in[9], in[11], in[13], in[15], out[9], out[11], out[13],
      39             :         out[15]);
      40           0 : }
      41             : 
      42           0 : static INLINE void transpose_16x16(const __m128i *in, __m128i *out) {
      43             :     // Upper left 8x8
      44           0 :     TRANSPOSE_4X4(in[0], in[4], in[8], in[12], out[0], out[4], out[8], out[12]);
      45           0 :     TRANSPOSE_4X4(in[1], in[5], in[9], in[13], out[16], out[20], out[24],
      46             :         out[28]);
      47           0 :     TRANSPOSE_4X4(in[16], in[20], in[24], in[28], out[1], out[5], out[9],
      48             :         out[13]);
      49           0 :     TRANSPOSE_4X4(in[17], in[21], in[25], in[29], out[17], out[21], out[25],
      50             :         out[29]);
      51             : 
      52             :     // Upper right 8x8
      53           0 :     TRANSPOSE_4X4(in[2], in[6], in[10], in[14], out[32], out[36], out[40],
      54             :         out[44]);
      55           0 :     TRANSPOSE_4X4(in[3], in[7], in[11], in[15], out[48], out[52], out[56],
      56             :         out[60]);
      57           0 :     TRANSPOSE_4X4(in[18], in[22], in[26], in[30], out[33], out[37], out[41],
      58             :         out[45]);
      59           0 :     TRANSPOSE_4X4(in[19], in[23], in[27], in[31], out[49], out[53], out[57],
      60             :         out[61]);
      61             : 
      62             :     // Lower left 8x8
      63           0 :     TRANSPOSE_4X4(in[32], in[36], in[40], in[44], out[2], out[6], out[10],
      64             :         out[14]);
      65           0 :     TRANSPOSE_4X4(in[33], in[37], in[41], in[45], out[18], out[22], out[26],
      66             :         out[30]);
      67           0 :     TRANSPOSE_4X4(in[48], in[52], in[56], in[60], out[3], out[7], out[11],
      68             :         out[15]);
      69           0 :     TRANSPOSE_4X4(in[49], in[53], in[57], in[61], out[19], out[23], out[27],
      70             :         out[31]);
      71             :     // Lower right 8x8
      72           0 :     TRANSPOSE_4X4(in[34], in[38], in[42], in[46], out[34], out[38], out[42],
      73             :         out[46]);
      74           0 :     TRANSPOSE_4X4(in[35], in[39], in[43], in[47], out[50], out[54], out[58],
      75             :         out[62]);
      76           0 :     TRANSPOSE_4X4(in[50], in[54], in[58], in[62], out[35], out[39], out[43],
      77             :         out[47]);
      78           0 :     TRANSPOSE_4X4(in[51], in[55], in[59], in[63], out[51], out[55], out[59],
      79             :         out[63]);
      80           0 : }
      81             : 
      82             : // Note:
      83             : //  rounding = 1 << (bit - 1)
      84           0 : static INLINE __m128i half_btf_sse4_1(const __m128i *w0, const __m128i *n0,
      85             :     const __m128i *w1, const __m128i *n1,
      86             :     const __m128i *rounding, int32_t bit) {
      87             :     __m128i x, y;
      88             : 
      89           0 :     x = _mm_mullo_epi32(*w0, *n0);
      90           0 :     y = _mm_mullo_epi32(*w1, *n1);
      91           0 :     x = _mm_add_epi32(x, y);
      92           0 :     x = _mm_add_epi32(x, *rounding);
      93           0 :     x = _mm_srai_epi32(x, bit);
      94           0 :     return x;
      95             : }
      96             : 
      97           0 : static INLINE __m128i half_btf_0_sse4_1(const __m128i *w0, const __m128i *n0,
      98             :     const __m128i *rounding, int32_t bit) {
      99             :     __m128i x;
     100             : 
     101           0 :     x = _mm_mullo_epi32(*w0, *n0);
     102           0 :     x = _mm_add_epi32(x, *rounding);
     103           0 :     x = _mm_srai_epi32(x, bit);
     104           0 :     return x;
     105             : }
     106             : 
     107             : #endif  // _HIGHBD_TXFM_UTILITY_SSE4_H