xref: /aosp_15_r20/external/libvpx/vpx_dsp/mips/variance_mmi.c (revision fb1b10ab9aebc7c7068eedab379b749d7e3900be)

/*
 *  Copyright (c) 2017 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/variance.h"
#include "vpx_ports/mem.h"
#include "vpx/vpx_integer.h"
#include "vpx_ports/asmdefs_mmi.h"

static const uint8_t bilinear_filters[8][2] = {
  { 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 },
  { 64, 64 }, { 48, 80 },  { 32, 96 }, { 16, 112 },
};

/* Use VARIANCE_SSE_SUM_8_FOR_W64 in vpx_variance64x64,vpx_variance64x32,
   vpx_variance32x64. VARIANCE_SSE_SUM_8 will lead to sum overflow. */
#define VARIANCE_SSE_SUM_8_FOR_W64                                  \
  /* sse */                                                         \
  "pasubub    %[ftmp3],   %[ftmp1],       %[ftmp2]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp3],       %[ftmp0]            \n\t" \
  "pmaddhw    %[ftmp6],   %[ftmp4],       %[ftmp4]            \n\t" \
  "pmaddhw    %[ftmp7],   %[ftmp5],       %[ftmp5]            \n\t" \
  "paddw      %[ftmp10],  %[ftmp10],      %[ftmp6]            \n\t" \
  "paddw      %[ftmp10],  %[ftmp10],      %[ftmp7]            \n\t" \
                                                                    \
  /* sum */                                                         \
  "punpcklbh  %[ftmp3],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp4],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpcklbh  %[ftmp5],   %[ftmp2],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp6],   %[ftmp2],       %[ftmp0]            \n\t" \
  "punpcklhw  %[ftmp1],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpckhhw  %[ftmp2],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpcklhw  %[ftmp7],   %[ftmp5],       %[ftmp0]            \n\t" \
  "punpckhhw  %[ftmp8],   %[ftmp5],       %[ftmp0]            \n\t" \
  "psubw      %[ftmp3],   %[ftmp1],       %[ftmp7]            \n\t" \
  "psubw      %[ftmp5],   %[ftmp2],       %[ftmp8]            \n\t" \
  "punpcklhw  %[ftmp1],   %[ftmp4],       %[ftmp0]            \n\t" \
  "punpckhhw  %[ftmp2],   %[ftmp4],       %[ftmp0]            \n\t" \
  "punpcklhw  %[ftmp7],   %[ftmp6],       %[ftmp0]            \n\t" \
  "punpckhhw  %[ftmp8],   %[ftmp6],       %[ftmp0]            \n\t" \
  "psubw      %[ftmp4],   %[ftmp1],       %[ftmp7]            \n\t" \
  "psubw      %[ftmp6],   %[ftmp2],       %[ftmp8]            \n\t" \
  "paddw      %[ftmp9],   %[ftmp9],       %[ftmp3]            \n\t" \
  "paddw      %[ftmp9],   %[ftmp9],       %[ftmp4]            \n\t" \
  "paddw      %[ftmp9],   %[ftmp9],       %[ftmp5]            \n\t" \
  "paddw      %[ftmp9],   %[ftmp9],       %[ftmp6]            \n\t"

#define VARIANCE_SSE_SUM_4                                          \
  /* sse */                                                         \
  "pasubub    %[ftmp3],   %[ftmp1],       %[ftmp2]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp3],       %[ftmp0]            \n\t" \
  "pmaddhw    %[ftmp5],   %[ftmp4],       %[ftmp4]            \n\t" \
  "paddw      %[ftmp6],   %[ftmp6],       %[ftmp5]            \n\t" \
                                                                    \
  /* sum */                                                         \
  "punpcklbh  %[ftmp3],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp2],       %[ftmp0]            \n\t" \
  "paddh      %[ftmp7],   %[ftmp7],       %[ftmp3]            \n\t" \
  "paddh      %[ftmp8],   %[ftmp8],       %[ftmp4]            \n\t"

#define VARIANCE_SSE_SUM_8                                          \
  /* sse */                                                         \
  "pasubub    %[ftmp3],   %[ftmp1],       %[ftmp2]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp3],       %[ftmp0]            \n\t" \
  "pmaddhw    %[ftmp6],   %[ftmp4],       %[ftmp4]            \n\t" \
  "pmaddhw    %[ftmp7],   %[ftmp5],       %[ftmp5]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp6]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp7]            \n\t" \
                                                                    \
  /* sum */                                                         \
  "punpcklbh  %[ftmp3],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp4],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpcklbh  %[ftmp5],   %[ftmp2],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp6],   %[ftmp2],       %[ftmp0]            \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ftmp3]            \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ftmp4]            \n\t" \
  "paddh      %[ftmp12],  %[ftmp12],      %[ftmp5]            \n\t" \
  "paddh      %[ftmp12],  %[ftmp12],      %[ftmp6]            \n\t"

#define VARIANCE_SSE_8                                              \
  "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t" \
  "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t" \
  "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t" \
  "pasubub    %[ftmp3],   %[ftmp1],       %[ftmp2]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp3],       %[ftmp0]            \n\t" \
  "pmaddhw    %[ftmp6],   %[ftmp4],       %[ftmp4]            \n\t" \
  "pmaddhw    %[ftmp7],   %[ftmp5],       %[ftmp5]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp6]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp7]            \n\t"

#define VARIANCE_SSE_16                                             \
  VARIANCE_SSE_8                                                    \
  "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "gsldlc1    %[ftmp2],   0x0f(%[ref_ptr])                    \n\t" \
  "gsldrc1    %[ftmp2],   0x08(%[ref_ptr])                    \n\t" \
  "pasubub    %[ftmp3],   %[ftmp1],       %[ftmp2]            \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp3],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp3],       %[ftmp0]            \n\t" \
  "pmaddhw    %[ftmp6],   %[ftmp4],       %[ftmp4]            \n\t" \
  "pmaddhw    %[ftmp7],   %[ftmp5],       %[ftmp5]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp6]            \n\t" \
  "paddw      %[ftmp8],   %[ftmp8],       %[ftmp7]            \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_A                       \
  /* calculate fdata3[0]~fdata3[3], store at ftmp2*/                \
  "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp2],   %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x01(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp3],   %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp2],   %[ftmp2],       %[filter_x0]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp3],   %[ftmp3],       %[filter_x1]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ftmp3]            \n\t" \
  "psrlh      %[ftmp2],   %[ftmp2],       %[ftmp6]            \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_B                       \
  /* calculate fdata3[0]~fdata3[3], store at ftmp4*/                \
  "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x01(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp5],   %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp4],   %[ftmp4],       %[filter_x0]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp5],   %[ftmp5],       %[filter_x1]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ftmp5]            \n\t" \
  "psrlh      %[ftmp4],   %[ftmp4],       %[ftmp6]            \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_A                      \
  /* calculate: temp2[0] ~ temp2[3] */                              \
  "pmullh     %[ftmp2],   %[ftmp2],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp4],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp2],   %[ftmp2],       %[ftmp6]            \n\t" \
                                                                    \
  /* store: temp2[0] ~ temp2[3] */                                  \
  "pand       %[ftmp2],   %[ftmp2],       %[mask]             \n\t" \
  "packushb   %[ftmp2],   %[ftmp2],       %[ftmp0]            \n\t" \
  "gssdrc1    %[ftmp2],   0x00(%[temp2_ptr])                  \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_B                      \
  /* calculate: temp2[0] ~ temp2[3] */                              \
  "pmullh     %[ftmp4],   %[ftmp4],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp2],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp4],   %[ftmp4],       %[ftmp6]            \n\t" \
                                                                    \
  /* store: temp2[0] ~ temp2[3] */                                  \
  "pand       %[ftmp4],   %[ftmp4],       %[mask]             \n\t" \
  "packushb   %[ftmp4],   %[ftmp4],       %[ftmp0]            \n\t" \
  "gssdrc1    %[ftmp4],   0x00(%[temp2_ptr])                  \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A                       \
  /* calculate fdata3[0]~fdata3[7], store at ftmp2 and ftmp3*/      \
  "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp2],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp3],   %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x01(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp2],   %[ftmp2],       %[filter_x0]        \n\t" \
  "pmullh     %[ftmp3],   %[ftmp3],       %[filter_x0]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ff_ph_40]         \n\t" \
  "paddh      %[ftmp3],   %[ftmp3],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp4],   %[ftmp4],       %[filter_x1]        \n\t" \
  "pmullh     %[ftmp5],   %[ftmp5],       %[filter_x1]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ftmp4]            \n\t" \
  "paddh      %[ftmp3],   %[ftmp3],       %[ftmp5]            \n\t" \
  "psrlh      %[ftmp2],   %[ftmp2],       %[ftmp14]           \n\t" \
  "psrlh      %[ftmp3],   %[ftmp3],       %[ftmp14]           \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_B                       \
  /* calculate fdata3[0]~fdata3[7], store at ftmp8 and ftmp9*/      \
  "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp8],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp9],   %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x01(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp10],  %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp11],  %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp8],   %[ftmp8],       %[filter_x0]        \n\t" \
  "pmullh     %[ftmp9],   %[ftmp9],       %[filter_x0]        \n\t" \
  "paddh      %[ftmp8],   %[ftmp8],       %[ff_ph_40]         \n\t" \
  "paddh      %[ftmp9],   %[ftmp9],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp10],  %[ftmp10],      %[filter_x1]        \n\t" \
  "pmullh     %[ftmp11],  %[ftmp11],      %[filter_x1]        \n\t" \
  "paddh      %[ftmp8],   %[ftmp8],       %[ftmp10]           \n\t" \
  "paddh      %[ftmp9],   %[ftmp9],       %[ftmp11]           \n\t" \
  "psrlh      %[ftmp8],   %[ftmp8],       %[ftmp14]           \n\t" \
  "psrlh      %[ftmp9],   %[ftmp9],       %[ftmp14]           \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_A                      \
  /* calculate: temp2[0] ~ temp2[3] */                              \
  "pmullh     %[ftmp2],   %[ftmp2],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp8],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp2],   %[ftmp2],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp2],   %[ftmp2],       %[ftmp14]           \n\t" \
                                                                    \
  /* calculate: temp2[4] ~ temp2[7] */                              \
  "pmullh     %[ftmp3],   %[ftmp3],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp3],   %[ftmp3],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp9],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp3],   %[ftmp3],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp3],   %[ftmp3],       %[ftmp14]           \n\t" \
                                                                    \
  /* store: temp2[0] ~ temp2[7] */                                  \
  "pand       %[ftmp2],   %[ftmp2],       %[mask]             \n\t" \
  "pand       %[ftmp3],   %[ftmp3],       %[mask]             \n\t" \
  "packushb   %[ftmp2],   %[ftmp2],       %[ftmp3]            \n\t" \
  "gssdlc1    %[ftmp2],   0x07(%[temp2_ptr])                  \n\t" \
  "gssdrc1    %[ftmp2],   0x00(%[temp2_ptr])                  \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_B                      \
  /* calculate: temp2[0] ~ temp2[3] */                              \
  "pmullh     %[ftmp8],   %[ftmp8],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp8],   %[ftmp8],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp2],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp8],   %[ftmp8],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp8],   %[ftmp8],       %[ftmp14]           \n\t" \
                                                                    \
  /* calculate: temp2[4] ~ temp2[7] */                              \
  "pmullh     %[ftmp9],   %[ftmp9],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp9],   %[ftmp9],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp3],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp9],   %[ftmp9],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp9],   %[ftmp9],       %[ftmp14]           \n\t" \
                                                                    \
  /* store: temp2[0] ~ temp2[7] */                                  \
  "pand       %[ftmp8],   %[ftmp8],       %[mask]             \n\t" \
  "pand       %[ftmp9],   %[ftmp9],       %[mask]             \n\t" \
  "packushb   %[ftmp8],   %[ftmp8],       %[ftmp9]            \n\t" \
  "gssdlc1    %[ftmp8],   0x07(%[temp2_ptr])                  \n\t" \
  "gssdrc1    %[ftmp8],   0x00(%[temp2_ptr])                  \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_A                      \
  /* calculate fdata3[0]~fdata3[7], store at ftmp2 and ftmp3*/      \
  VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A                             \
                                                                    \
  /* calculate fdata3[8]~fdata3[15], store at ftmp4 and ftmp5*/     \
  "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp4],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp5],   %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x10(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x09(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp6],   %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp7],   %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp4],   %[ftmp4],       %[filter_x0]        \n\t" \
  "pmullh     %[ftmp5],   %[ftmp5],       %[filter_x0]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ff_ph_40]         \n\t" \
  "paddh      %[ftmp5],   %[ftmp5],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp6],   %[ftmp6],       %[filter_x1]        \n\t" \
  "pmullh     %[ftmp7],   %[ftmp7],       %[filter_x1]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ftmp6]            \n\t" \
  "paddh      %[ftmp5],   %[ftmp5],       %[ftmp7]            \n\t" \
  "psrlh      %[ftmp4],   %[ftmp4],       %[ftmp14]           \n\t" \
  "psrlh      %[ftmp5],   %[ftmp5],       %[ftmp14]           \n\t"

#define VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_B                      \
  /* calculate fdata3[0]~fdata3[7], store at ftmp8 and ftmp9*/      \
  VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_B                             \
                                                                    \
  /* calculate fdata3[8]~fdata3[15], store at ftmp10 and ftmp11*/   \
  "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp10],  %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp11],  %[ftmp1],       %[ftmp0]            \n\t" \
  "gsldlc1    %[ftmp1],   0x10(%[src_ptr])                    \n\t" \
  "gsldrc1    %[ftmp1],   0x09(%[src_ptr])                    \n\t" \
  "punpcklbh  %[ftmp12],  %[ftmp1],       %[ftmp0]            \n\t" \
  "punpckhbh  %[ftmp13],  %[ftmp1],       %[ftmp0]            \n\t" \
  "pmullh     %[ftmp10],  %[ftmp10],      %[filter_x0]        \n\t" \
  "pmullh     %[ftmp11],  %[ftmp11],      %[filter_x0]        \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ff_ph_40]         \n\t" \
  "paddh      %[ftmp11],  %[ftmp11],      %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp12],  %[ftmp12],      %[filter_x1]        \n\t" \
  "pmullh     %[ftmp13],  %[ftmp13],      %[filter_x1]        \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ftmp12]           \n\t" \
  "paddh      %[ftmp11],  %[ftmp11],      %[ftmp13]           \n\t" \
  "psrlh      %[ftmp10],  %[ftmp10],      %[ftmp14]           \n\t" \
  "psrlh      %[ftmp11],  %[ftmp11],      %[ftmp14]           \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_A                     \
  VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_A                            \
                                                                    \
  /* calculate: temp2[8] ~ temp2[11] */                             \
  "pmullh     %[ftmp4],   %[ftmp4],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp10],      %[filter_y1]        \n\t" \
  "paddh      %[ftmp4],   %[ftmp4],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp4],   %[ftmp4],       %[ftmp14]           \n\t" \
                                                                    \
  /* calculate: temp2[12] ~ temp2[15] */                            \
  "pmullh     %[ftmp5],   %[ftmp5],       %[filter_y0]        \n\t" \
  "paddh      %[ftmp5],   %[ftmp5],       %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp11],       %[filter_y1]       \n\t" \
  "paddh      %[ftmp5],   %[ftmp5],       %[ftmp1]            \n\t" \
  "psrlh      %[ftmp5],   %[ftmp5],       %[ftmp14]           \n\t" \
                                                                    \
  /* store: temp2[8] ~ temp2[15] */                                 \
  "pand       %[ftmp4],   %[ftmp4],       %[mask]             \n\t" \
  "pand       %[ftmp5],   %[ftmp5],       %[mask]             \n\t" \
  "packushb   %[ftmp4],   %[ftmp4],       %[ftmp5]            \n\t" \
  "gssdlc1    %[ftmp4],   0x0f(%[temp2_ptr])                  \n\t" \
  "gssdrc1    %[ftmp4],   0x08(%[temp2_ptr])                  \n\t"

#define VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_B                     \
  VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_B                            \
                                                                    \
  /* calculate: temp2[8] ~ temp2[11] */                             \
  "pmullh     %[ftmp10],  %[ftmp10],      %[filter_y0]        \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp4],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp10],  %[ftmp10],      %[ftmp1]            \n\t" \
  "psrlh      %[ftmp10],  %[ftmp10],      %[ftmp14]           \n\t" \
                                                                    \
  /* calculate: temp2[12] ~ temp2[15] */                            \
  "pmullh     %[ftmp11],  %[ftmp11],      %[filter_y0]        \n\t" \
  "paddh      %[ftmp11],  %[ftmp11],      %[ff_ph_40]         \n\t" \
  "pmullh     %[ftmp1],   %[ftmp5],       %[filter_y1]        \n\t" \
  "paddh      %[ftmp11],  %[ftmp11],      %[ftmp1]            \n\t" \
  "psrlh      %[ftmp11],  %[ftmp11],      %[ftmp14]           \n\t" \
                                                                    \
  /* store: temp2[8] ~ temp2[15] */                                 \
  "pand       %[ftmp10],  %[ftmp10],      %[mask]             \n\t" \
  "pand       %[ftmp11],  %[ftmp11],      %[mask]             \n\t" \
  "packushb   %[ftmp10],  %[ftmp10],      %[ftmp11]           \n\t" \
  "gssdlc1    %[ftmp10],  0x0f(%[temp2_ptr])                  \n\t" \
  "gssdrc1    %[ftmp10],  0x08(%[temp2_ptr])                  \n\t"

// Applies a 1-D 2-tap bilinear filter to the source block in either horizontal
// or vertical direction to produce the filtered output block. Used to implement
// the first-pass of 2-D separable filter.
//
// Produces int16_t output to retain precision for the next pass. Two filter
// taps should sum to FILTER_WEIGHT. pixel_step defines whether the filter is
// applied horizontally (pixel_step = 1) or vertically (pixel_step = stride).
// It defines the offset required to move from one input to the next.
static void var_filter_block2d_bil_first_pass(
    const uint8_t *src_ptr, uint16_t *ref_ptr, unsigned int src_pixels_per_line,
    int pixel_step, unsigned int output_height, unsigned int output_width,
    const uint8_t *filter) {
  unsigned int i, j;

  for (i = 0; i < output_height; ++i) {
    for (j = 0; j < output_width; ++j) {
      ref_ptr[j] = ROUND_POWER_OF_TWO(
          (int)src_ptr[0] * filter[0] + (int)src_ptr[pixel_step] * filter[1],
          FILTER_BITS);

      ++src_ptr;
    }

    src_ptr += src_pixels_per_line - output_width;
    ref_ptr += output_width;
  }
}

// Applies a 1-D 2-tap bilinear filter to the source block in either horizontal
// or vertical direction to produce the filtered output block. Used to implement
// the second-pass of 2-D separable filter.
//
// Requires 16-bit input as produced by filter_block2d_bil_first_pass. Two
// filter taps should sum to FILTER_WEIGHT. pixel_step defines whether the
// filter is applied horizontally (pixel_step = 1) or vertically
// (pixel_step = stride). It defines the offset required to move from one input
// to the next. Output is 8-bit.
static void var_filter_block2d_bil_second_pass(
    const uint16_t *src_ptr, uint8_t *ref_ptr, unsigned int src_pixels_per_line,
    unsigned int pixel_step, unsigned int output_height,
    unsigned int output_width, const uint8_t *filter) {
  unsigned int i, j;

  for (i = 0; i < output_height; ++i) {
    for (j = 0; j < output_width; ++j) {
      ref_ptr[j] = ROUND_POWER_OF_TWO(
          (int)src_ptr[0] * filter[0] + (int)src_ptr[pixel_step] * filter[1],
          FILTER_BITS);
      ++src_ptr;
    }

    src_ptr += src_pixels_per_line - output_width;
    ref_ptr += output_width;
  }
}

static inline uint32_t vpx_variance64x(const uint8_t *src_ptr, int src_stride,
                                       const uint8_t *ref_ptr, int ref_stride,
                                       uint32_t *sse, int high) {
  int sum;
  double ftmp[12];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp9],   %[ftmp9],       %[ftmp9]            \n\t"
    "pxor       %[ftmp10],  %[ftmp10],      %[ftmp10]           \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x0f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x08(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x17(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x10(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x17(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x10(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x1f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x18(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x1f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x18(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x27(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x20(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x27(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x20(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x2f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x28(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x2f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x28(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x37(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x30(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x37(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x30(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x3f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x38(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x3f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x38(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "mfc1       %[tmp1],    %[ftmp9]                            \n\t"
    "mfhc1      %[tmp2],    %[ftmp9]                            \n\t"
    "addu       %[sum],     %[tmp1],        %[tmp2]             \n\t"
    "ssrld      %[ftmp1],   %[ftmp10],      %[ftmp11]           \n\t"
    "paddw      %[ftmp1],   %[ftmp1],       %[ftmp10]           \n\t"
    "swc1       %[ftmp1],   0x00(%[sse])                        \n\t"
    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
      [tmp2]"=&r"(tmp[2]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr),
      [sum]"=&r"(sum)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / (64 * high));
}

#define VPX_VARIANCE64XN(n)                                                   \
  uint32_t vpx_variance64x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                                    const uint8_t *ref_ptr, int ref_stride,   \
                                    uint32_t *sse) {                          \
    return vpx_variance64x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

VPX_VARIANCE64XN(64)
VPX_VARIANCE64XN(32)

uint32_t vpx_variance32x64_mmi(const uint8_t *src_ptr, int src_stride,
                               const uint8_t *ref_ptr, int ref_stride,
                               uint32_t *sse) {
  int sum;
  double ftmp[12];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    "li         %[tmp0],    0x40                                \n\t"
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp9],   %[ftmp9],       %[ftmp9]            \n\t"
    "pxor       %[ftmp10],  %[ftmp10],      %[ftmp10]           \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x0f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x08(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x17(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x10(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x17(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x10(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "gsldlc1    %[ftmp1],   0x1f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x18(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x1f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x18(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8_FOR_W64

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "mfc1       %[tmp1],    %[ftmp9]                            \n\t"
    "mfhc1      %[tmp2],    %[ftmp9]                            \n\t"
    "addu       %[sum],     %[tmp1],        %[tmp2]             \n\t"
    "ssrld      %[ftmp1],   %[ftmp10],      %[ftmp11]           \n\t"
    "paddw      %[ftmp1],   %[ftmp1],       %[ftmp10]           \n\t"
    "swc1       %[ftmp1],   0x00(%[sse])                        \n\t"
    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
      [tmp2]"=&r"(tmp[2]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr),
      [sum]"=&r"(sum)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [sse]"r"(sse)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / 2048);
}

static inline uint32_t vpx_variance32x(const uint8_t *src_ptr, int src_stride,
                                       const uint8_t *ref_ptr, int ref_stride,
                                       uint32_t *sse, int high) {
  int sum;
  double ftmp[13];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"
    "pxor       %[ftmp10],  %[ftmp10],      %[ftmp10]           \n\t"
    "pxor       %[ftmp12],  %[ftmp12],      %[ftmp12]           \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8
    "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x0f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x08(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8
    "gsldlc1    %[ftmp1],   0x17(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x10(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x17(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x10(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8
    "gsldlc1    %[ftmp1],   0x1f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x18(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x1f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x18(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp8],       %[ftmp11]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp8]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"

    "punpcklhw  %[ftmp3],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp4],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpcklhw  %[ftmp5],   %[ftmp12],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp6],   %[ftmp12],      %[ftmp0]            \n\t"
    "paddw      %[ftmp3],   %[ftmp3],       %[ftmp4]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp5]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp6]            \n\t"
    "ssrld      %[ftmp0],   %[ftmp3],       %[ftmp11]           \n\t"
    "paddw      %[ftmp0],   %[ftmp0],       %[ftmp3]            \n\t"
    "swc1       %[ftmp0],   0x00(%[sum])                        \n\t"

    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [ftmp12]"=&f"(ftmp[12]),          [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / (32 * high));
}

#define VPX_VARIANCE32XN(n)                                                   \
  uint32_t vpx_variance32x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                                    const uint8_t *ref_ptr, int ref_stride,   \
                                    uint32_t *sse) {                          \
    return vpx_variance32x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

VPX_VARIANCE32XN(32)
VPX_VARIANCE32XN(16)

static inline uint32_t vpx_variance16x(const uint8_t *src_ptr, int src_stride,
                                       const uint8_t *ref_ptr, int ref_stride,
                                       uint32_t *sse, int high) {
  int sum;
  double ftmp[13];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"
    "pxor       %[ftmp10],  %[ftmp10],      %[ftmp10]           \n\t"
    "pxor       %[ftmp12],  %[ftmp12],      %[ftmp12]           \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8
    "gsldlc1    %[ftmp1],   0x0f(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x08(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x0f(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x08(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp8],       %[ftmp11]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp8]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"

    "punpcklhw  %[ftmp3],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp4],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpcklhw  %[ftmp5],   %[ftmp12],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp6],   %[ftmp12],      %[ftmp0]            \n\t"
    "paddw      %[ftmp3],   %[ftmp3],       %[ftmp4]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp5]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp6]            \n\t"
    "ssrld      %[ftmp0],   %[ftmp3],       %[ftmp11]           \n\t"
    "paddw      %[ftmp0],   %[ftmp0],       %[ftmp3]            \n\t"
    "swc1       %[ftmp0],   0x00(%[sum])                        \n\t"

    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [ftmp12]"=&f"(ftmp[12]),          [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / (16 * high));
}

#define VPX_VARIANCE16XN(n)                                                   \
  uint32_t vpx_variance16x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                                    const uint8_t *ref_ptr, int ref_stride,   \
                                    uint32_t *sse) {                          \
    return vpx_variance16x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

VPX_VARIANCE16XN(32)
VPX_VARIANCE16XN(16)
VPX_VARIANCE16XN(8)

static inline uint32_t vpx_variance8x(const uint8_t *src_ptr, int src_stride,
                                      const uint8_t *ref_ptr, int ref_stride,
                                      uint32_t *sse, int high) {
  int sum;
  double ftmp[13];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"
    "pxor       %[ftmp10],  %[ftmp10],      %[ftmp10]           \n\t"
    "pxor       %[ftmp12],  %[ftmp12],      %[ftmp12]           \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_8

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp8],       %[ftmp11]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp8]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"

    "punpcklhw  %[ftmp3],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp4],   %[ftmp10],      %[ftmp0]            \n\t"
    "punpcklhw  %[ftmp5],   %[ftmp12],      %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp6],   %[ftmp12],      %[ftmp0]            \n\t"
    "paddw      %[ftmp3],   %[ftmp3],       %[ftmp4]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp5]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp6]            \n\t"
    "ssrld      %[ftmp0],   %[ftmp3],       %[ftmp11]           \n\t"
    "paddw      %[ftmp0],   %[ftmp0],       %[ftmp3]            \n\t"
    "swc1       %[ftmp0],   0x00(%[sum])                        \n\t"

    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [ftmp12]"=&f"(ftmp[12]),          [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / (8 * high));
}

#define VPX_VARIANCE8XN(n)                                                   \
  uint32_t vpx_variance8x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                                   const uint8_t *ref_ptr, int ref_stride,   \
                                   uint32_t *sse) {                          \
    return vpx_variance8x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

VPX_VARIANCE8XN(16)
VPX_VARIANCE8XN(8)
VPX_VARIANCE8XN(4)

static inline uint32_t vpx_variance4x(const uint8_t *src_ptr, int src_stride,
                                      const uint8_t *ref_ptr, int ref_stride,
                                      uint32_t *sse, int high) {
  int sum;
  double ftmp[12];
  uint32_t tmp[3];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp10]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp6],   %[ftmp6],       %[ftmp6]            \n\t"
    "pxor       %[ftmp7],   %[ftmp7],       %[ftmp7]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"
    "1:                                                         \n\t"
    "gsldlc1    %[ftmp1],   0x07(%[src_ptr])                    \n\t"
    "gsldrc1    %[ftmp1],   0x00(%[src_ptr])                    \n\t"
    "gsldlc1    %[ftmp2],   0x07(%[ref_ptr])                    \n\t"
    "gsldrc1    %[ftmp2],   0x00(%[ref_ptr])                    \n\t"
    VARIANCE_SSE_SUM_4

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp6],       %[ftmp10]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp6]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"

    "punpcklhw  %[ftmp3],   %[ftmp7],       %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp4],   %[ftmp7],       %[ftmp0]            \n\t"
    "punpcklhw  %[ftmp5],   %[ftmp8],       %[ftmp0]            \n\t"
    "punpckhhw  %[ftmp6],   %[ftmp8],       %[ftmp0]            \n\t"
    "paddw      %[ftmp3],   %[ftmp3],       %[ftmp4]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp5]            \n\t"
    "psubw      %[ftmp3],   %[ftmp3],       %[ftmp6]            \n\t"
    "ssrld      %[ftmp0],   %[ftmp3],       %[ftmp10]           \n\t"
    "paddw      %[ftmp0],   %[ftmp0],       %[ftmp3]            \n\t"
    "swc1       %[ftmp0],   0x00(%[sum])                        \n\t"
    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),
      [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse), [sum]"r"(&sum)
    : "memory"
  );
  /* clang-format on */

  return *sse - (((int64_t)sum * sum) / (4 * high));
}

#define VPX_VARIANCE4XN(n)                                                   \
  uint32_t vpx_variance4x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                                   const uint8_t *ref_ptr, int ref_stride,   \
                                   uint32_t *sse) {                          \
    return vpx_variance4x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

VPX_VARIANCE4XN(8)
VPX_VARIANCE4XN(4)

static inline uint32_t vpx_mse16x(const uint8_t *src_ptr, int src_stride,
                                  const uint8_t *ref_ptr, int ref_stride,
                                  uint32_t *sse, uint64_t high) {
  double ftmp[12];
  uint32_t tmp[1];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"

    "1:                                                         \n\t"
    VARIANCE_SSE_16

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp8],       %[ftmp11]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp8]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"
    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse)
    : "memory"
  );
  /* clang-format on */

  return *sse;
}

#define vpx_mse16xN(n)                                                   \
  uint32_t vpx_mse16x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                               const uint8_t *ref_ptr, int ref_stride,   \
                               uint32_t *sse) {                          \
    return vpx_mse16x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

vpx_mse16xN(16);
vpx_mse16xN(8);

static inline uint32_t vpx_mse8x(const uint8_t *src_ptr, int src_stride,
                                 const uint8_t *ref_ptr, int ref_stride,
                                 uint32_t *sse, uint64_t high) {
  double ftmp[12];
  uint32_t tmp[1];

  *sse = 0;

  /* clang-format off */
  __asm__ volatile (
    "li         %[tmp0],    0x20                                \n\t"
    "mtc1       %[tmp0],    %[ftmp11]                           \n\t"
    MMI_L(%[tmp0], %[high], 0x00)
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    "pxor       %[ftmp8],   %[ftmp8],       %[ftmp8]            \n\t"

    "1:                                                         \n\t"
    VARIANCE_SSE_8

    "addiu      %[tmp0],    %[tmp0],        -0x01               \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    MMI_ADDU(%[ref_ptr], %[ref_ptr], %[ref_stride])
    "bnez       %[tmp0],    1b                                  \n\t"

    "ssrld      %[ftmp9],   %[ftmp8],       %[ftmp11]           \n\t"
    "paddw      %[ftmp9],   %[ftmp9],       %[ftmp8]            \n\t"
    "swc1       %[ftmp9],   0x00(%[sse])                        \n\t"
    : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
      [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
      [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
      [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
      [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
      [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
      [tmp0]"=&r"(tmp[0]),
      [src_ptr]"+&r"(src_ptr),          [ref_ptr]"+&r"(ref_ptr)
    : [src_stride]"r"((mips_reg)src_stride),
      [ref_stride]"r"((mips_reg)ref_stride),
      [high]"r"(&high), [sse]"r"(sse)
    : "memory"
  );
  /* clang-format on */

  return *sse;
}

#define vpx_mse8xN(n)                                                   \
  uint32_t vpx_mse8x##n##_mmi(const uint8_t *src_ptr, int src_stride,   \
                              const uint8_t *ref_ptr, int ref_stride,   \
                              uint32_t *sse) {                          \
    return vpx_mse8x(src_ptr, src_stride, ref_ptr, ref_stride, sse, n); \
  }

vpx_mse8xN(16);
vpx_mse8xN(8);

#define SUBPIX_VAR(W, H)                                                       \
  uint32_t vpx_sub_pixel_variance##W##x##H##_mmi(                              \
      const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset,      \
      const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) {                 \
    uint16_t fdata3[((H) + 1) * (W)];                                          \
    uint8_t temp2[(H) * (W)];                                                  \
                                                                               \
    var_filter_block2d_bil_first_pass(src_ptr, fdata3, src_stride, 1, (H) + 1, \
                                      W, bilinear_filters[x_offset]);          \
    var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W,              \
                                       bilinear_filters[y_offset]);            \
                                                                               \
    return vpx_variance##W##x##H##_mmi(temp2, W, ref_ptr, ref_stride, sse);    \
  }

SUBPIX_VAR(64, 64)
SUBPIX_VAR(64, 32)
SUBPIX_VAR(32, 64)
SUBPIX_VAR(32, 32)
SUBPIX_VAR(32, 16)
SUBPIX_VAR(16, 32)

static inline void var_filter_block2d_bil_16x(const uint8_t *src_ptr,
                                              int src_stride, int x_offset,
                                              int y_offset, uint8_t *temp2,
                                              int counter) {
  uint8_t *temp2_ptr = temp2;
  mips_reg l_counter = counter;
  double ftmp[15];
  double ff_ph_40, mask;
  double filter_x0, filter_x1, filter_y0, filter_y1;
  mips_reg tmp[2];
  uint64_t x0, x1, y0, y1, all;

  const uint8_t *filter_x = bilinear_filters[x_offset];
  const uint8_t *filter_y = bilinear_filters[y_offset];
  x0 = (uint64_t)filter_x[0];
  x1 = (uint64_t)filter_x[1];
  y0 = (uint64_t)filter_y[0];
  y1 = (uint64_t)filter_y[1];
  all = x0 | x1 << 8 | y0 << 16 | y1 << 24;

  /* clang-format off */
  __asm__ volatile (
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    MMI_MTC1(%[all], %[ftmp14])
    "punpcklbh  %[ftmp14],  %[ftmp14],      %[ftmp0]            \n\t"
    "pshufh     %[filter_x0], %[ftmp14],    %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x10)
    MMI_MTC1(%[tmp0], %[mask])
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_x1], %[ftmp14],    %[ftmp0]            \n\t"
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_y0], %[ftmp14],    %[ftmp0]            \n\t"
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_y1], %[ftmp14],    %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x07)
    MMI_MTC1(%[tmp0], %[ftmp14])
    MMI_LI(%[tmp0], 0x0040004000400040)
    MMI_MTC1(%[tmp0], %[ff_ph_40])
    MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
    MMI_MTC1(%[tmp0], %[mask])
    // fdata3: fdata3[0] ~ fdata3[15]
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_A

    // fdata3 +src_stride*1: fdata3[0] ~ fdata3[15]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_B
    // temp2: temp2[0] ~ temp2[15]
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_A

    // fdata3 +src_stride*2: fdata3[0] ~ fdata3[15]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_A
    // temp2+16*1: temp2[0] ~ temp2[15]
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x10)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_B

    "1:                                                         \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_B
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x10)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_A

    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_16_A
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x10)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_16_B
    "addiu      %[counter], %[counter],     -0x01               \n\t"
    "bnez       %[counter], 1b                                  \n\t"
    : [ftmp0] "=&f"(ftmp[0]), [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2]),
      [ftmp3] "=&f"(ftmp[3]), [ftmp4] "=&f"(ftmp[4]), [ftmp5] "=&f"(ftmp[5]),
      [ftmp6] "=&f"(ftmp[6]), [ftmp7] "=&f"(ftmp[7]), [ftmp8] "=&f"(ftmp[8]),
      [ftmp9] "=&f"(ftmp[9]), [ftmp10] "=&f"(ftmp[10]),
      [ftmp11] "=&f"(ftmp[11]), [ftmp12] "=&f"(ftmp[12]),
      [ftmp13] "=&f"(ftmp[13]), [ftmp14] "=&f"(ftmp[14]),
      [tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr), [temp2_ptr] "+&r"(temp2_ptr),
      [counter]"+&r"(l_counter), [ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
      [filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
      [filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
    : [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
    : "memory"
  );
  /* clang-format on */
}

#define SUBPIX_VAR16XN(H)                                                      \
  uint32_t vpx_sub_pixel_variance16x##H##_mmi(                                 \
      const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset,      \
      const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) {                 \
    uint8_t temp2[16 * (H)];                                                   \
    var_filter_block2d_bil_16x(src_ptr, src_stride, x_offset, y_offset, temp2, \
                               ((H)-2) / 2);                                   \
                                                                               \
    return vpx_variance16x##H##_mmi(temp2, 16, ref_ptr, ref_stride, sse);      \
  }

SUBPIX_VAR16XN(16)
SUBPIX_VAR16XN(8)

static inline void var_filter_block2d_bil_8x(const uint8_t *src_ptr,
                                             int src_stride, int x_offset,
                                             int y_offset, uint8_t *temp2,
                                             int counter) {
  uint8_t *temp2_ptr = temp2;
  mips_reg l_counter = counter;
  double ftmp[15];
  mips_reg tmp[2];
  double ff_ph_40, mask;
  uint64_t x0, x1, y0, y1, all;
  double filter_x0, filter_x1, filter_y0, filter_y1;
  const uint8_t *filter_x = bilinear_filters[x_offset];
  const uint8_t *filter_y = bilinear_filters[y_offset];
  x0 = (uint64_t)filter_x[0];
  x1 = (uint64_t)filter_x[1];
  y0 = (uint64_t)filter_y[0];
  y1 = (uint64_t)filter_y[1];
  all = x0 | x1 << 8 | y0 << 16 | y1 << 24;

  /* clang-format off */
  __asm__ volatile (
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    MMI_MTC1(%[all], %[ftmp14])
    "punpcklbh  %[ftmp14],  %[ftmp14],      %[ftmp0]            \n\t"
    "pshufh     %[filter_x0], %[ftmp14],    %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x10)
    MMI_MTC1(%[tmp0], %[mask])
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_x1], %[ftmp14],    %[ftmp0]            \n\t"
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_y0], %[ftmp14],    %[ftmp0]            \n\t"
    "ssrld      %[ftmp14],  %[ftmp14],      %[mask]             \n\t"
    "pshufh     %[filter_y1], %[ftmp14],    %[ftmp0]            \n\t"
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x07)
    MMI_MTC1(%[tmp0], %[ftmp14])
    MMI_LI(%[tmp0], 0x0040004000400040)
    MMI_MTC1(%[tmp0], %[ff_ph_40])
    MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
    MMI_MTC1(%[tmp0], %[mask])

    // fdata3: fdata3[0] ~ fdata3[7]
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A

    // fdata3 +src_stride*1: fdata3[0] ~ fdata3[7]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_B
    // temp2: temp2[0] ~ temp2[7]
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_A

    // fdata3 +src_stride*2: fdata3[0] ~ fdata3[7]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A
    // temp2+8*1: temp2[0] ~ temp2[7]
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x08)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_B

    "1:                                                         \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_B
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x08)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_A

    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_8_A
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x08)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_8_B
    "addiu      %[counter], %[counter],     -0x01               \n\t"
    "bnez       %[counter], 1b                                  \n\t"
    : [ftmp0] "=&f"(ftmp[0]), [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2]),
      [ftmp3] "=&f"(ftmp[3]), [ftmp4] "=&f"(ftmp[4]), [ftmp5] "=&f"(ftmp[5]),
      [ftmp6] "=&f"(ftmp[6]), [ftmp7] "=&f"(ftmp[7]), [ftmp8] "=&f"(ftmp[8]),
      [ftmp9] "=&f"(ftmp[9]), [ftmp10] "=&f"(ftmp[10]),
      [ftmp11] "=&f"(ftmp[11]), [ftmp12] "=&f"(ftmp[12]),
      [ftmp13] "=&f"(ftmp[13]), [ftmp14] "=&f"(ftmp[14]),
      [tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr), [temp2_ptr] "+&r"(temp2_ptr),
      [counter]"+&r"(l_counter), [ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
      [filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
      [filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
    : [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
    : "memory"
  );
  /* clang-format on */
}

#define SUBPIX_VAR8XN(H)                                                      \
  uint32_t vpx_sub_pixel_variance8x##H##_mmi(                                 \
      const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset,     \
      const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) {                \
    uint8_t temp2[8 * (H)];                                                   \
    var_filter_block2d_bil_8x(src_ptr, src_stride, x_offset, y_offset, temp2, \
                              ((H)-2) / 2);                                   \
                                                                              \
    return vpx_variance8x##H##_mmi(temp2, 8, ref_ptr, ref_stride, sse);       \
  }

SUBPIX_VAR8XN(16)
SUBPIX_VAR8XN(8)
SUBPIX_VAR8XN(4)

static inline void var_filter_block2d_bil_4x(const uint8_t *src_ptr,
                                             int src_stride, int x_offset,
                                             int y_offset, uint8_t *temp2,
                                             int counter) {
  uint8_t *temp2_ptr = temp2;
  mips_reg l_counter = counter;
  double ftmp[7];
  mips_reg tmp[2];
  double ff_ph_40, mask;
  uint64_t x0, x1, y0, y1, all;
  double filter_x0, filter_x1, filter_y0, filter_y1;
  const uint8_t *filter_x = bilinear_filters[x_offset];
  const uint8_t *filter_y = bilinear_filters[y_offset];
  x0 = (uint64_t)filter_x[0];
  x1 = (uint64_t)filter_x[1];
  y0 = (uint64_t)filter_y[0];
  y1 = (uint64_t)filter_y[1];
  all = x0 | x1 << 8 | y0 << 16 | y1 << 24;

  /* clang-format off */
  __asm__ volatile (
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    MMI_MTC1(%[all], %[ftmp6])
    "punpcklbh  %[ftmp6],   %[ftmp6],       %[ftmp0]            \n\t"
    "pshufh     %[filter_x0], %[ftmp6],     %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x10)
    MMI_MTC1(%[tmp0], %[mask])
    "ssrld      %[ftmp6],   %[ftmp6],       %[mask]             \n\t"
    "pshufh     %[filter_x1], %[ftmp6],     %[ftmp0]            \n\t"
    "ssrld      %[ftmp6],   %[ftmp6],       %[mask]             \n\t"
    "pshufh     %[filter_y0], %[ftmp6],     %[ftmp0]            \n\t"
    "ssrld      %[ftmp6],   %[ftmp6],       %[mask]             \n\t"
    "pshufh     %[filter_y1], %[ftmp6],     %[ftmp0]            \n\t"
    "pxor       %[ftmp0],   %[ftmp0],       %[ftmp0]            \n\t"
    MMI_LI(%[tmp0], 0x07)
    MMI_MTC1(%[tmp0], %[ftmp6])
    MMI_LI(%[tmp0], 0x0040004000400040)
    MMI_MTC1(%[tmp0], %[ff_ph_40])
    MMI_LI(%[tmp0], 0x00ff00ff00ff00ff)
    MMI_MTC1(%[tmp0], %[mask])
    // fdata3: fdata3[0] ~ fdata3[3]
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_A

    // fdata3 +src_stride*1: fdata3[0] ~ fdata3[3]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_B
    // temp2: temp2[0] ~ temp2[7]
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_A

    // fdata3 +src_stride*2: fdata3[0] ~ fdata3[3]
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_A
    // temp2+4*1: temp2[0] ~ temp2[7]
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x04)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_B

    "1:                                                         \n\t"
    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_B
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x04)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_A

    MMI_ADDU(%[src_ptr], %[src_ptr], %[src_stride])
    VAR_FILTER_BLOCK2D_BIL_FIRST_PASS_4_A
    MMI_ADDIU(%[temp2_ptr], %[temp2_ptr], 0x04)
    VAR_FILTER_BLOCK2D_BIL_SECOND_PASS_4_B
    "addiu      %[counter], %[counter],     -0x01               \n\t"
    "bnez       %[counter], 1b                                  \n\t"
    : [ftmp0] "=&f"(ftmp[0]), [ftmp1] "=&f"(ftmp[1]), [ftmp2] "=&f"(ftmp[2]),
      [ftmp3] "=&f"(ftmp[3]), [ftmp4] "=&f"(ftmp[4]), [ftmp5] "=&f"(ftmp[5]),
      [ftmp6] "=&f"(ftmp[6]), [tmp0] "=&r"(tmp[0]), [src_ptr] "+&r"(src_ptr),
      [temp2_ptr] "+&r"(temp2_ptr), [counter]"+&r"(l_counter),
      [ff_ph_40] "=&f"(ff_ph_40), [mask] "=&f"(mask),
      [filter_x0] "=&f"(filter_x0), [filter_x1] "=&f"(filter_x1),
      [filter_y0] "=&f"(filter_y0), [filter_y1] "=&f"(filter_y1)
    : [src_stride] "r"((mips_reg)src_stride), [all] "r"(all)
    : "memory"
  );
  /* clang-format on */
}

#define SUBPIX_VAR4XN(H)                                                      \
  uint32_t vpx_sub_pixel_variance4x##H##_mmi(                                 \
      const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset,     \
      const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) {                \
    uint8_t temp2[4 * (H)];                                                   \
    var_filter_block2d_bil_4x(src_ptr, src_stride, x_offset, y_offset, temp2, \
                              ((H)-2) / 2);                                   \
                                                                              \
    return vpx_variance4x##H##_mmi(temp2, 4, ref_ptr, ref_stride, sse);       \
  }

SUBPIX_VAR4XN(8)
SUBPIX_VAR4XN(4)

#define SUBPIX_AVG_VAR(W, H)                                                   \
  uint32_t vpx_sub_pixel_avg_variance##W##x##H##_mmi(                          \
      const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset,      \
      const uint8_t *ref_ptr, int ref_stride, uint32_t *sse,                   \
      const uint8_t *second_pred) {                                            \
    uint16_t fdata3[((H) + 1) * (W)];                                          \
    uint8_t temp2[(H) * (W)];                                                  \
    DECLARE_ALIGNED(16, uint8_t, temp3[(H) * (W)]);                            \
                                                                               \
    var_filter_block2d_bil_first_pass(src_ptr, fdata3, src_stride, 1, (H) + 1, \
                                      W, bilinear_filters[x_offset]);          \
    var_filter_block2d_bil_second_pass(fdata3, temp2, W, W, H, W,              \
                                       bilinear_filters[y_offset]);            \
                                                                               \
    vpx_comp_avg_pred_c(temp3, second_pred, W, H, temp2, W);                   \
                                                                               \
    return vpx_variance##W##x##H##_mmi(temp3, W, ref_ptr, ref_stride, sse);    \
  }

SUBPIX_AVG_VAR(64, 64)
SUBPIX_AVG_VAR(64, 32)
SUBPIX_AVG_VAR(32, 64)
SUBPIX_AVG_VAR(32, 32)
SUBPIX_AVG_VAR(32, 16)
SUBPIX_AVG_VAR(16, 32)
SUBPIX_AVG_VAR(16, 16)
SUBPIX_AVG_VAR(16, 8)
SUBPIX_AVG_VAR(8, 16)
SUBPIX_AVG_VAR(8, 8)
SUBPIX_AVG_VAR(8, 4)
SUBPIX_AVG_VAR(4, 8)
SUBPIX_AVG_VAR(4, 4)