x264 source for verification 2026-05-22
This commit is contained in:
259
common/x86/util.h
Normal file
259
common/x86/util.h
Normal file
@@ -0,0 +1,259 @@
|
||||
/*****************************************************************************
|
||||
* util.h: x86 inline asm
|
||||
*****************************************************************************
|
||||
* Copyright (C) 2008-2025 x264 project
|
||||
*
|
||||
* Authors: Fiona Glaser <fiona@x264.com>
|
||||
* Loren Merritt <lorenm@u.washington.edu>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
|
||||
*
|
||||
* This program is also available under a commercial proprietary license.
|
||||
* For more information, contact us at licensing@x264.com.
|
||||
*****************************************************************************/
|
||||
|
||||
#ifndef X264_X86_UTIL_H
|
||||
#define X264_X86_UTIL_H
|
||||
|
||||
#ifdef __SSE__
|
||||
#include <xmmintrin.h>
|
||||
|
||||
#undef M128_ZERO
|
||||
#define M128_ZERO ((__m128){0,0,0,0})
|
||||
#define x264_union128_t x264_union128_sse_t
|
||||
typedef union { __m128 i; uint64_t q[2]; uint32_t d[4]; uint16_t w[8]; uint8_t b[16]; } MAY_ALIAS x264_union128_sse_t;
|
||||
#if HAVE_VECTOREXT
|
||||
typedef uint32_t v4si __attribute__((vector_size (16)));
|
||||
#endif
|
||||
#endif // __SSE__
|
||||
|
||||
#if HAVE_X86_INLINE_ASM && HAVE_MMX
|
||||
|
||||
#define x264_median_mv x264_median_mv_mmx2
|
||||
static ALWAYS_INLINE void x264_median_mv_mmx2( int16_t *dst, int16_t *a, int16_t *b, int16_t *c )
|
||||
{
|
||||
asm(
|
||||
"movd %1, %%mm0 \n"
|
||||
"movd %2, %%mm1 \n"
|
||||
"movq %%mm0, %%mm3 \n"
|
||||
"movd %3, %%mm2 \n"
|
||||
"pmaxsw %%mm1, %%mm0 \n"
|
||||
"pminsw %%mm3, %%mm1 \n"
|
||||
"pminsw %%mm2, %%mm0 \n"
|
||||
"pmaxsw %%mm1, %%mm0 \n"
|
||||
"movd %%mm0, %0 \n"
|
||||
:"=m"(*(x264_union32_t*)dst)
|
||||
:"m"(M32( a )), "m"(M32( b )), "m"(M32( c ))
|
||||
:"mm0", "mm1", "mm2", "mm3"
|
||||
);
|
||||
}
|
||||
|
||||
#define x264_predictor_difference x264_predictor_difference_mmx2
|
||||
static ALWAYS_INLINE int x264_predictor_difference_mmx2( int16_t (*mvc)[2], intptr_t i_mvc )
|
||||
{
|
||||
int sum;
|
||||
static const uint64_t pw_1 = 0x0001000100010001ULL;
|
||||
|
||||
asm(
|
||||
"pxor %%mm4, %%mm4 \n"
|
||||
"test $1, %1 \n"
|
||||
"jnz 3f \n"
|
||||
"movd -8(%2,%1,4), %%mm0 \n"
|
||||
"movd -4(%2,%1,4), %%mm3 \n"
|
||||
"psubw %%mm3, %%mm0 \n"
|
||||
"jmp 2f \n"
|
||||
"3: \n"
|
||||
"dec %1 \n"
|
||||
"1: \n"
|
||||
"movq -8(%2,%1,4), %%mm0 \n"
|
||||
"psubw -4(%2,%1,4), %%mm0 \n"
|
||||
"2: \n"
|
||||
"sub $2, %1 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"psubw %%mm0, %%mm2 \n"
|
||||
"pmaxsw %%mm2, %%mm0 \n"
|
||||
"paddusw %%mm0, %%mm4 \n"
|
||||
"jg 1b \n"
|
||||
"pmaddwd %4, %%mm4 \n"
|
||||
"pshufw $14, %%mm4, %%mm0 \n"
|
||||
"paddd %%mm0, %%mm4 \n"
|
||||
"movd %%mm4, %0 \n"
|
||||
:"=r"(sum), "+r"(i_mvc)
|
||||
:"r"(mvc), "m"(MEM_DYN( mvc, const int16_t )), "m"(pw_1)
|
||||
:"mm0", "mm2", "mm3", "mm4", "cc"
|
||||
);
|
||||
return sum;
|
||||
}
|
||||
|
||||
#define x264_cabac_mvd_sum x264_cabac_mvd_sum_mmx2
|
||||
static ALWAYS_INLINE uint16_t x264_cabac_mvd_sum_mmx2(uint8_t *mvdleft, uint8_t *mvdtop)
|
||||
{
|
||||
static const uint64_t pb_2 = 0x0202020202020202ULL;
|
||||
static const uint64_t pb_32 = 0x2020202020202020ULL;
|
||||
static const uint64_t pb_33 = 0x2121212121212121ULL;
|
||||
int amvd;
|
||||
asm(
|
||||
"movd %1, %%mm0 \n"
|
||||
"movd %2, %%mm1 \n"
|
||||
"paddusb %%mm1, %%mm0 \n"
|
||||
"pminub %5, %%mm0 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"movq %%mm0, %%mm1 \n"
|
||||
"pcmpgtb %3, %%mm0 \n"
|
||||
"pcmpgtb %4, %%mm1 \n"
|
||||
"psubb %%mm0, %%mm2 \n"
|
||||
"psubb %%mm1, %%mm2 \n"
|
||||
"movd %%mm2, %0 \n"
|
||||
:"=r"(amvd)
|
||||
:"m"(M16( mvdleft )),"m"(M16( mvdtop )),
|
||||
"m"(pb_2),"m"(pb_32),"m"(pb_33)
|
||||
:"mm0", "mm1", "mm2"
|
||||
);
|
||||
return (uint16_t)amvd;
|
||||
}
|
||||
|
||||
#define x264_predictor_clip x264_predictor_clip_mmx2
|
||||
static ALWAYS_INLINE int x264_predictor_clip_mmx2( int16_t (*dst)[2], int16_t (*mvc)[2], int i_mvc, int16_t mv_limit[2][2], uint32_t pmv )
|
||||
{
|
||||
static const uint32_t pd_32 = 0x20;
|
||||
intptr_t tmp = (intptr_t)mv_limit, mvc_max = i_mvc, i = 0;
|
||||
|
||||
asm(
|
||||
"movq (%2), %%mm5 \n"
|
||||
"movd %6, %%mm3 \n"
|
||||
"psllw $2, %%mm5 \n" // Convert to subpel
|
||||
"pshufw $0xEE, %%mm5, %%mm6 \n"
|
||||
"dec %k3 \n"
|
||||
"jz 2f \n" // if( i_mvc == 1 ) {do the last iteration}
|
||||
"punpckldq %%mm3, %%mm3 \n"
|
||||
"punpckldq %%mm5, %%mm5 \n"
|
||||
"movd %7, %%mm4 \n"
|
||||
"lea (%0,%3,4), %3 \n"
|
||||
"1: \n"
|
||||
"movq (%0), %%mm0 \n"
|
||||
"add $8, %0 \n"
|
||||
"movq %%mm3, %%mm1 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"pcmpeqd %%mm0, %%mm1 \n" // mv == pmv
|
||||
"pcmpeqd %%mm0, %%mm2 \n" // mv == 0
|
||||
"por %%mm1, %%mm2 \n" // (mv == pmv || mv == 0) * -1
|
||||
"pmovmskb %%mm2, %k2 \n" // (mv == pmv || mv == 0) * 0xf
|
||||
"pmaxsw %%mm5, %%mm0 \n"
|
||||
"pminsw %%mm6, %%mm0 \n"
|
||||
"pand %%mm4, %%mm2 \n" // (mv0 == pmv || mv0 == 0) * 32
|
||||
"psrlq %%mm2, %%mm0 \n" // drop mv0 if it's skipped
|
||||
"movq %%mm0, (%5,%4,4) \n"
|
||||
"and $24, %k2 \n"
|
||||
"add $2, %4 \n"
|
||||
"add $8, %k2 \n"
|
||||
"shr $4, %k2 \n" // (4-val)>>1
|
||||
"sub %2, %4 \n" // +1 for each valid motion vector
|
||||
"cmp %3, %0 \n"
|
||||
"jl 1b \n"
|
||||
"jg 3f \n" // if( i == i_mvc - 1 ) {do the last iteration}
|
||||
|
||||
/* Do the last iteration */
|
||||
"2: \n"
|
||||
"movd (%0), %%mm0 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"pcmpeqd %%mm0, %%mm3 \n"
|
||||
"pcmpeqd %%mm0, %%mm2 \n"
|
||||
"por %%mm3, %%mm2 \n"
|
||||
"pmovmskb %%mm2, %k2 \n"
|
||||
"pmaxsw %%mm5, %%mm0 \n"
|
||||
"pminsw %%mm6, %%mm0 \n"
|
||||
"movd %%mm0, (%5,%4,4) \n"
|
||||
"inc %4 \n"
|
||||
"and $1, %k2 \n"
|
||||
"sub %2, %4 \n" // output += !(mv == pmv || mv == 0)
|
||||
"3: \n"
|
||||
:"+r"(mvc), "=m"(MEM_DYN( dst, int16_t )), "+r"(tmp), "+r"(mvc_max), "+r"(i)
|
||||
:"r"(dst), "g"(pmv), "m"(pd_32), "m"(MEM_DYN( mvc, const int16_t ))
|
||||
:"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "cc"
|
||||
);
|
||||
return i;
|
||||
}
|
||||
|
||||
/* Same as the above, except we do (mv + 2) >> 2 on the input. */
|
||||
#define x264_predictor_roundclip x264_predictor_roundclip_mmx2
|
||||
static ALWAYS_INLINE int x264_predictor_roundclip_mmx2( int16_t (*dst)[2], int16_t (*mvc)[2], int i_mvc, int16_t mv_limit[2][2], uint32_t pmv )
|
||||
{
|
||||
static const uint64_t pw_2 = 0x0002000200020002ULL;
|
||||
static const uint32_t pd_32 = 0x20;
|
||||
intptr_t tmp = (intptr_t)mv_limit, mvc_max = i_mvc, i = 0;
|
||||
|
||||
asm(
|
||||
"movq (%2), %%mm5 \n"
|
||||
"movq %6, %%mm7 \n"
|
||||
"movd %7, %%mm3 \n"
|
||||
"pshufw $0xEE, %%mm5, %%mm6 \n"
|
||||
"dec %k3 \n"
|
||||
"jz 2f \n"
|
||||
"punpckldq %%mm3, %%mm3 \n"
|
||||
"punpckldq %%mm5, %%mm5 \n"
|
||||
"movd %8, %%mm4 \n"
|
||||
"lea (%0,%3,4), %3 \n"
|
||||
"1: \n"
|
||||
"movq (%0), %%mm0 \n"
|
||||
"add $8, %0 \n"
|
||||
"paddw %%mm7, %%mm0 \n"
|
||||
"psraw $2, %%mm0 \n"
|
||||
"movq %%mm3, %%mm1 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"pcmpeqd %%mm0, %%mm1 \n"
|
||||
"pcmpeqd %%mm0, %%mm2 \n"
|
||||
"por %%mm1, %%mm2 \n"
|
||||
"pmovmskb %%mm2, %k2 \n"
|
||||
"pmaxsw %%mm5, %%mm0 \n"
|
||||
"pminsw %%mm6, %%mm0 \n"
|
||||
"pand %%mm4, %%mm2 \n"
|
||||
"psrlq %%mm2, %%mm0 \n"
|
||||
"movq %%mm0, (%5,%4,4) \n"
|
||||
"and $24, %k2 \n"
|
||||
"add $2, %4 \n"
|
||||
"add $8, %k2 \n"
|
||||
"shr $4, %k2 \n"
|
||||
"sub %2, %4 \n"
|
||||
"cmp %3, %0 \n"
|
||||
"jl 1b \n"
|
||||
"jg 3f \n"
|
||||
|
||||
/* Do the last iteration */
|
||||
"2: \n"
|
||||
"movd (%0), %%mm0 \n"
|
||||
"paddw %%mm7, %%mm0 \n"
|
||||
"psraw $2, %%mm0 \n"
|
||||
"pxor %%mm2, %%mm2 \n"
|
||||
"pcmpeqd %%mm0, %%mm3 \n"
|
||||
"pcmpeqd %%mm0, %%mm2 \n"
|
||||
"por %%mm3, %%mm2 \n"
|
||||
"pmovmskb %%mm2, %k2 \n"
|
||||
"pmaxsw %%mm5, %%mm0 \n"
|
||||
"pminsw %%mm6, %%mm0 \n"
|
||||
"movd %%mm0, (%5,%4,4) \n"
|
||||
"inc %4 \n"
|
||||
"and $1, %k2 \n"
|
||||
"sub %2, %4 \n"
|
||||
"3: \n"
|
||||
:"+r"(mvc), "=m"(MEM_DYN( dst, int16_t )), "+r"(tmp), "+r"(mvc_max), "+r"(i)
|
||||
:"r"(dst), "m"(pw_2), "g"(pmv), "m"(pd_32), "m"(MEM_DYN( mvc, const int16_t ))
|
||||
:"mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", "cc"
|
||||
);
|
||||
return i;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user