Line data    Source code

       1             : /*
       2             :  * Copyright (c) 2018, Alliance for Open Media. All rights reserved
       3             :  *
       4             :  * This source code is subject to the terms of the BSD 2 Clause License and
       5             :  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
       6             :  * was not distributed with this source code in the LICENSE file, you can
       7             :  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
       8             :  * Media Patent License 1.0 was not distributed with this source code in the
       9             :  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
      10             :  */
      11             : 
      12             : #include <tmmintrin.h>
      13             : 
      14             : // #include "config/aom_config.h"
      15             : #include "aom_dsp_rtcd.h"
      16             : 
      17             : #include "synonyms.h"
      18             : 
      19           0 : void aom_var_filter_block2d_bil_first_pass_ssse3(
      20             :     const uint8_t *a, uint16_t *b, unsigned int src_pixels_per_line,
      21             :     unsigned int pixel_step, unsigned int output_height,
      22             :     unsigned int output_width, const uint8_t *filter) {
      23             :   // Note: filter[0], filter[1] could be {128, 0}, where 128 will overflow
      24             :   // in computation using _mm_maddubs_epi16.
      25             :   // Change {128, 0} to {64, 0} and reduce FILTER_BITS by 1 to avoid overflow.
      26           0 :   const int16_t round = (1 << (FILTER_BITS - 1)) >> 1;
      27           0 :   const __m128i r = _mm_set1_epi16(round);
      28           0 :   const uint8_t f0 = filter[0] >> 1;
      29           0 :   const uint8_t f1 = filter[1] >> 1;
      30           0 :   const __m128i filters = _mm_setr_epi8(f0, f1, f0, f1, f0, f1, f0, f1, f0, f1,
      31             :                                         f0, f1, f0, f1, f0, f1);
      32             :   unsigned int i, j;
      33             :   (void)pixel_step;
      34             : 
      35           0 :   if (output_width >= 8) {
      36           0 :     for (i = 0; i < output_height; ++i) {
      37           0 :       for (j = 0; j < output_width; j += 8) {
      38             :         // load source
      39           0 :         __m128i source_low = xx_loadl_64(a);
      40           0 :         __m128i source_hi = xx_loadl_64(a + 1);
      41             : 
      42             :         // unpack to:
      43             :         // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
      44             :         //   a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
      45           0 :         __m128i source = _mm_unpacklo_epi8(source_low, source_hi);
      46             : 
      47             :         // b[i] = a[i] * filter[0] + a[i + 1] * filter[1]
      48           0 :         __m128i res = _mm_maddubs_epi16(source, filters);
      49             : 
      50             :         // round
      51           0 :         res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
      52             : 
      53           0 :         xx_storeu_128(b, res);
      54             : 
      55           0 :         a += 8;
      56           0 :         b += 8;
      57             :       }
      58             : 
      59           0 :       a += src_pixels_per_line - output_width;
      60             :     }
      61             :   } else {
      62             :     const __m128i shuffle_mask =
      63           0 :         _mm_setr_epi8(0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8);
      64           0 :     for (i = 0; i < output_height; ++i) {
      65             :       // load source, only first 5 values are meaningful:
      66             :       // { a[0], a[1], a[2], a[3], a[4], xxxx }
      67           0 :       __m128i source = xx_loadl_64(a);
      68             : 
      69             :       // shuffle, up to the first 8 are useful
      70             :       // { a[0], a[1], a[1], a[2], a[2], a[3], a[3], a[4],
      71             :       //   a[4], a[5], a[5], a[6], a[6], a[7], a[7], a[8] }
      72           0 :       __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
      73             : 
      74           0 :       __m128i res = _mm_maddubs_epi16(source_shuffle, filters);
      75           0 :       res = _mm_srai_epi16(_mm_add_epi16(res, r), FILTER_BITS - 1);
      76             : 
      77           0 :       xx_storel_64(b, res);
      78             : 
      79           0 :       a += src_pixels_per_line;
      80           0 :       b += output_width;
      81             :     }
      82             :   }
      83           0 : }
      84             : 
      85           0 : void aom_var_filter_block2d_bil_second_pass_ssse3(
      86             :     const uint16_t *a, uint8_t *b, unsigned int src_pixels_per_line,
      87             :     unsigned int pixel_step, unsigned int output_height,
      88             :     unsigned int output_width, const uint8_t *filter) {
      89           0 :   const int16_t round = (1 << FILTER_BITS) >> 1;
      90           0 :   const __m128i r = _mm_set1_epi32(round);
      91             :   const __m128i filters =
      92           0 :       _mm_setr_epi16(filter[0], filter[1], filter[0], filter[1], filter[0],
      93           0 :                      filter[1], filter[0], filter[1]);
      94             :   const __m128i shuffle_mask =
      95           0 :       _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
      96             :   const __m128i mask =
      97           0 :       _mm_setr_epi8(0, 4, 8, 12, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1);
      98             :   unsigned int i, j;
      99             : 
     100           0 :   for (i = 0; i < output_height; ++i) {
     101           0 :     for (j = 0; j < output_width; j += 4) {
     102             :       // load source as:
     103             :       // { a[0], a[1], a[2], a[3], a[w], a[w+1], a[w+2], a[w+3] }
     104           0 :       __m128i source1 = xx_loadl_64(a);
     105           0 :       __m128i source2 = xx_loadl_64(a + pixel_step);
     106           0 :       __m128i source = _mm_unpacklo_epi64(source1, source2);
     107             : 
     108             :       // shuffle source to:
     109             :       // { a[0], a[w], a[1], a[w+1], a[2], a[w+2], a[3], a[w+3] }
     110           0 :       __m128i source_shuffle = _mm_shuffle_epi8(source, shuffle_mask);
     111             : 
     112             :       // b[i] = a[i] * filter[0] + a[w + i] * filter[1]
     113           0 :       __m128i res = _mm_madd_epi16(source_shuffle, filters);
     114             : 
     115             :       // round
     116           0 :       res = _mm_srai_epi32(_mm_add_epi32(res, r), FILTER_BITS);
     117             : 
     118             :       // shuffle to get each lower 8 bit of every 32 bit
     119           0 :       res = _mm_shuffle_epi8(res, mask);
     120             : 
     121           0 :       xx_storel_32(b, res);
     122             : 
     123           0 :       a += 4;
     124           0 :       b += 4;
     125             :     }
     126             : 
     127           0 :     a += src_pixels_per_line - output_width;
     128             :   }
     129           0 : }