diff --git a/Makefile.am b/Makefile.am index 2a1ddc8..0188e11 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,6 +23,9 @@ SILK_SOURCES += $(SILK_SOURCES_SSE4_1) $(SILK_SOURCES_FIXED_SSE4_1) endif else SILK_SOURCES += $(SILK_SOURCES_FLOAT) +if HAVE_SSE4_1 +SILK_SOURCES += $(SILK_SOURCES_SSE4_1) +endif endif if DISABLE_FLOAT_API @@ -30,12 +33,14 @@ else OPUS_SOURCES += $(OPUS_SOURCES_FLOAT) endif -if HAVE_SSE4_1 -CELT_SOURCES += $(CELT_SOURCES_SSE) $(CELT_SOURCES_SSE4_1) -else -if HAVE_SSE2 +if HAVE_SSE CELT_SOURCES += $(CELT_SOURCES_SSE) endif +if HAVE_SSE2 +CELT_SOURCES += $(CELT_SOURCES_SSE2) +endif +if HAVE_SSE4_1 +CELT_SOURCES += $(CELT_SOURCES_SSE4_1) endif if CPU_ARM @@ -44,7 +49,6 @@ SILK_SOURCES += $(SILK_SOURCES_ARM) if OPUS_ARM_NEON_INTR CELT_SOURCES += $(CELT_SOURCES_ARM_NEON_INTR) -OPUS_ARM_NEON_INTR_CPPFLAGS = -mfpu=neon endif if OPUS_ARM_EXTERNAL_ASM @@ -258,18 +262,24 @@ $(CELT_SOURCES_ARM_ASM:%.s=%-gnu.S): $(top_srcdir)/celt/arm/arm2gnu.pl %-gnu.S: %.s $(top_srcdir)/celt/arm/arm2gnu.pl @ARM2GNU_PARAMS@ < $< > $@ -SSE_OBJ = %_sse.o %_sse.lo %test_unit_mathops.o %test_unit_rotation.o +OPT_UNIT_TEST_OBJ = $(celt_tests_test_unit_mathops_SOURCES:.c=.o) $(celt_tests_test_unit_rotation_SOURCES:.c=.o) + +if HAVE_SSE +SSE_OBJ = $(CELT_SOURCES_SSE:.c=.lo) +$(SSE_OBJ) $(OPT_UNIT_TEST_OBJ): CFLAGS += $(OPUS_X86_SSE_CFLAGS) +endif -if HAVE_SSE4_1 -$(SSE_OBJ): CFLAGS += -msse4.1 -else if HAVE_SSE2 -$(SSE_OBJ): CFLAGS += -msse2 +SSE2_OBJ = $(CELT_SOURCES_SSE2:.c=.lo) +$(SSE2_OBJ) $(OPT_UNIT_TEST_OBJ): CFLAGS += $(OPUS_X86_SSE2_CFLAGS) endif + +if HAVE_SSE4_1 +SSE4_1_OBJ = $(CELT_SOURCES_SSE4_1:.c=.lo) $(SILK_SOURCES_SSE4_1:.c=.lo) $(SILK_SOURCES_FIXED_SSE4_1:.c=.lo) +$(SSE4_1_OBJ) $(OPT_UNIT_TEST_OBJ): CFLAGS += $(OPUS_X86_SSE4_1_CFLAGS) endif if OPUS_ARM_NEON_INTR -CELT_ARM_NEON_INTR_OBJ = $(CELT_SOURCES_ARM_NEON_INTR:.c=.lo) \ - %test_unit_rotation.o %test_unit_mathops.o -$(CELT_ARM_NEON_INTR_OBJ): CFLAGS += $(OPUS_ARM_NEON_INTR_CPPFLAGS) +CELT_ARM_NEON_INTR_OBJ = $(CELT_SOURCES_ARM_NEON_INTR:.c=.lo) +$(CELT_ARM_NEON_INTR_OBJ) $(OPT_UNIT_TEST_OBJ): CFLAGS += $(OPUS_ARM_NEON_INTR_CFLAGS) endif diff --git a/celt/arm/armcpu.c b/celt/arm/armcpu.c index 1768525..5e5d10c 100644 --- a/celt/arm/armcpu.c +++ b/celt/arm/armcpu.c @@ -73,7 +73,7 @@ static OPUS_INLINE opus_uint32 opus_cpu_capabilities(void){ __except(GetExceptionCode()==EXCEPTION_ILLEGAL_INSTRUCTION){ /*Ignore exception.*/ } -# if defined(OPUS_ARM_MAY_HAVE_NEON) +# if defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) __try{ /*VORR q0,q0,q0*/ __emit(0xF2200150); @@ -107,7 +107,7 @@ opus_uint32 opus_cpu_capabilities(void) while(fgets(buf, 512, cpuinfo) != NULL) { -# if defined(OPUS_ARM_MAY_HAVE_EDSP) || defined(OPUS_ARM_MAY_HAVE_NEON) +# if defined(OPUS_ARM_MAY_HAVE_EDSP) || defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) /* Search for edsp and neon flag */ if(memcmp(buf, "Features", 8) == 0) { @@ -118,7 +118,7 @@ opus_uint32 opus_cpu_capabilities(void) flags |= OPUS_CPU_ARM_EDSP; # endif -# if defined(OPUS_ARM_MAY_HAVE_NEON) +# if defined(OPUS_ARM_MAY_HAVE_NEON) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR) p = strstr(buf, " neon"); if(p != NULL && (p[5] == ' ' || p[5] == '\n')) flags |= OPUS_CPU_ARM_NEON; diff --git a/celt/arm/pitch_arm.h b/celt/arm/pitch_arm.h index 125d1bc..8626ed7 100644 --- a/celt/arm/pitch_arm.h +++ b/celt/arm/pitch_arm.h @@ -54,10 +54,10 @@ opus_val32 celt_pitch_xcorr_edsp(const opus_val16 *_x, const opus_val16 *_y, #else /* Start !FIXED_POINT */ /* Float case */ -#if defined(OPUS_ARM_NEON_INTR) +#if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) void celt_pitch_xcorr_float_neon(const opus_val16 *_x, const opus_val16 *_y, opus_val32 *xcorr, int len, int max_pitch); -#if !defined(OPUS_HAVE_RTCD) +#if !defined(OPUS_HAVE_RTCD) || defined(OPUS_ARM_PRESUME_NEON_INTR) #define OVERRIDE_PITCH_XCORR (1) # define celt_pitch_xcorr(_x, _y, xcorr, len, max_pitch, arch) \ ((void)(arch),celt_pitch_xcorr_float_neon(_x, _y, xcorr, len, max_pitch)) diff --git a/celt/bands.c b/celt/bands.c index c643b09..25f229e 100644 --- a/celt/bands.c +++ b/celt/bands.c @@ -398,7 +398,7 @@ static void stereo_split(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT } } -static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, opus_val16 mid, int N) +static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, opus_val16 mid, int N, int arch) { int j; opus_val32 xp=0, side=0; @@ -410,7 +410,7 @@ static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT opus_val32 t, lgain, rgain; /* Compute the norm of X+Y and X-Y as |X|^2 + |Y|^2 +/- sum(xy) */ - dual_inner_prod(Y, X, Y, N, &xp, &side); + dual_inner_prod(Y, X, Y, N, &xp, &side, arch); /* Compensating for the mid normalization */ xp = MULT16_32_Q15(mid, xp); /* mid and side are in Q15, not Q14 like X and Y */ @@ -1348,7 +1348,7 @@ static unsigned quant_band_stereo(struct band_ctx *ctx, celt_norm *X, celt_norm if (resynth) { if (N!=2) - stereo_merge(X, Y, mid, N); + stereo_merge(X, Y, mid, N, ctx->arch); if (inv) { int j; diff --git a/celt/celt.c b/celt/celt.c index a610de4..cd70cb1 100644 --- a/celt/celt.c +++ b/celt/celt.c @@ -89,10 +89,12 @@ int resampling_factor(opus_int32 rate) return ret; } -#ifndef OVERRIDE_COMB_FILTER_CONST /* This version should be faster on ARM */ #ifdef OPUS_ARM_ASM -static void comb_filter_const(opus_val32 *y, opus_val32 *x, int T, int N, +#ifndef NON_STATIC_COMB_FILTER_CONST_C +static +#endif +void comb_filter_const_c(opus_val32 *y, opus_val32 *x, int T, int N, opus_val16 g10, opus_val16 g11, opus_val16 g12) { opus_val32 x0, x1, x2, x3, x4; @@ -147,7 +149,10 @@ static void comb_filter_const(opus_val32 *y, opus_val32 *x, int T, int N, #endif } #else -static void comb_filter_const(opus_val32 *y, opus_val32 *x, int T, int N, +#ifndef NON_STATIC_COMB_FILTER_CONST_C +static +#endif +void comb_filter_const_c(opus_val32 *y, opus_val32 *x, int T, int N, opus_val16 g10, opus_val16 g11, opus_val16 g12) { opus_val32 x0, x1, x2, x3, x4; @@ -171,12 +176,16 @@ static void comb_filter_const(opus_val32 *y, opus_val32 *x, int T, int N, } #endif + +#ifndef OVERRIDE_COMB_FILTER_CONST +# define comb_filter_const(y, x, T, N, g10, g11, g12, arch) \ + ((void)(arch),comb_filter_const_c(y, x, T, N, g10, g11, g12)) #endif #ifndef OVERRIDE_comb_filter void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, opus_val16 g0, opus_val16 g1, int tapset0, int tapset1, - const opus_val16 *window, int overlap) + const opus_val16 *window, int overlap, int arch) { int i; /* printf ("%d %d %f %f\n", T0, T1, g0, g1); */ @@ -234,7 +243,7 @@ void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, } /* Compute the part with the constant filter. */ - comb_filter_const(y+i, x+i, T1, N-i, g10, g11, g12); + comb_filter_const(y+i, x+i, T1, N-i, g10, g11, g12, arch); } #endif /* OVERRIDE_comb_filter */ diff --git a/celt/celt.h b/celt/celt.h index b196751..5a281fe 100644 --- a/celt/celt.h +++ b/celt/celt.h @@ -201,7 +201,7 @@ void celt_preemphasis(const opus_val16 * OPUS_RESTRICT pcmp, celt_sig * OPUS_RES void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, opus_val16 g0, opus_val16 g1, int tapset0, int tapset1, - const opus_val16 *window, int overlap); + const opus_val16 *window, int overlap, int arch); void init_caps(const CELTMode *m,int *cap,int LM,int C); diff --git a/celt/celt_decoder.c b/celt/celt_decoder.c index 4304a3e..3f4be80 100644 --- a/celt/celt_decoder.c +++ b/celt/celt_decoder.c @@ -698,7 +698,7 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) comb_filter(etmp, buf+DECODE_BUFFER_SIZE, st->postfilter_period, st->postfilter_period, overlap, -st->postfilter_gain, -st->postfilter_gain, - st->postfilter_tapset, st->postfilter_tapset, NULL, 0); + st->postfilter_tapset, st->postfilter_tapset, NULL, 0, st->arch); /* Simulate TDAC on the concealed audio so that it blends with the MDCT of the next frame. */ @@ -1009,11 +1009,11 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat st->postfilter_period_old=IMAX(st->postfilter_period_old, COMBFILTER_MINPERIOD); comb_filter(out_syn[c], out_syn[c], st->postfilter_period_old, st->postfilter_period, mode->shortMdctSize, st->postfilter_gain_old, st->postfilter_gain, st->postfilter_tapset_old, st->postfilter_tapset, - mode->window, overlap); + mode->window, overlap, st->arch); if (LM!=0) comb_filter(out_syn[c]+mode->shortMdctSize, out_syn[c]+mode->shortMdctSize, st->postfilter_period, postfilter_pitch, N-mode->shortMdctSize, st->postfilter_gain, postfilter_gain, st->postfilter_tapset, postfilter_tapset, - mode->window, overlap); + mode->window, overlap, st->arch); } while (++cpostfilter_period_old = st->postfilter_period; diff --git a/celt/celt_encoder.c b/celt/celt_encoder.c index 86a3fbb..07e7071 100644 --- a/celt/celt_encoder.c +++ b/celt/celt_encoder.c @@ -1163,11 +1163,11 @@ static int run_prefilter(CELTEncoder *st, celt_sig *in, celt_sig *prefilter_mem, if (offset) comb_filter(in+c*(N+overlap)+overlap, pre[c]+COMBFILTER_MAXPERIOD, st->prefilter_period, st->prefilter_period, offset, -st->prefilter_gain, -st->prefilter_gain, - st->prefilter_tapset, st->prefilter_tapset, NULL, 0); + st->prefilter_tapset, st->prefilter_tapset, NULL, 0, st->arch); comb_filter(in+c*(N+overlap)+overlap+offset, pre[c]+COMBFILTER_MAXPERIOD+offset, st->prefilter_period, pitch_index, N-offset, -st->prefilter_gain, -gain1, - st->prefilter_tapset, prefilter_tapset, mode->window, overlap); + st->prefilter_tapset, prefilter_tapset, mode->window, overlap, st->arch); OPUS_COPY(st->in_mem+c*(overlap), in+c*(N+overlap)+N, overlap); if (N>COMBFILTER_MAXPERIOD) diff --git a/celt/celt_lpc.h b/celt/celt_lpc.h index dc8967f..323459e 100644 --- a/celt/celt_lpc.h +++ b/celt/celt_lpc.h @@ -48,7 +48,7 @@ void celt_fir_c( opus_val16 *mem, int arch); -#if !defined(OPUS_X86_MAY_HAVE_SSE4_1) +#if !defined(OVERRIDE_CELT_FIR) #define celt_fir(x, num, y, N, ord, mem, arch) \ (celt_fir_c(x, num, y, N, ord, mem, arch)) #endif diff --git a/celt/cpu_support.h b/celt/cpu_support.h index 1d62e2f..8bf7ffc 100644 --- a/celt/cpu_support.h +++ b/celt/cpu_support.h @@ -43,14 +43,16 @@ */ #define OPUS_ARCHMASK 3 -#elif defined(OPUS_X86_MAY_HAVE_SSE2) || defined(OPUS_X86_MAY_HAVE_SSE4_1) +#elif (defined(OPUS_X86_MAY_HAVE_SSE) && !defined(OPUS_X86_PRESUME_SSE)) || \ + (defined(OPUS_X86_MAY_HAVE_SSE2) && !defined(OPUS_X86_PRESUME_SSE2)) || \ + (defined(OPUS_X86_MAY_HAVE_SSE4_1) && !defined(OPUS_X86_PRESUME_SSE4_1)) #include "x86/x86cpu.h" -/* We currently support 3 x86 variants: +/* We currently support 4 x86 variants: * arch[0] -> non-sse - * arch[1] -> sse2 - * arch[2] -> sse4.1 - * arch[3] -> NULL + * arch[1] -> sse + * arch[2] -> sse2 + * arch[3] -> sse4.1 */ #define OPUS_ARCHMASK 3 int opus_select_arch(void); diff --git a/celt/mips/celt_mipsr1.h b/celt/mips/celt_mipsr1.h index 03915d8..7915d59 100644 --- a/celt/mips/celt_mipsr1.h +++ b/celt/mips/celt_mipsr1.h @@ -56,7 +56,7 @@ #define OVERRIDE_comb_filter void comb_filter(opus_val32 *y, opus_val32 *x, int T0, int T1, int N, opus_val16 g0, opus_val16 g1, int tapset0, int tapset1, - const opus_val16 *window, int overlap) + const opus_val16 *window, int overlap, int arch) { int i; opus_val32 x0, x1, x2, x3, x4; diff --git a/celt/pitch.c b/celt/pitch.c index 4364703..1d89cb0 100644 --- a/celt/pitch.c +++ b/celt/pitch.c @@ -439,7 +439,7 @@ opus_val16 remove_doubling(opus_val16 *x, int maxperiod, int minperiod, T = T0 = *T0_; ALLOC(yy_lookup, maxperiod+1, opus_val32); - dual_inner_prod(x, x, x-T0, N, &xx, &xy); + dual_inner_prod(x, x, x-T0, N, &xx, &xy, arch); yy_lookup[0] = xx; yy=xx; for (i=1;i<=maxperiod;i++) @@ -483,7 +483,7 @@ opus_val16 remove_doubling(opus_val16 *x, int maxperiod, int minperiod, { T1b = celt_udiv(2*second_check[k]*T0+k, 2*k); } - dual_inner_prod(x, &x[-T1], &x[-T1b], N, &xy, &xy2); + dual_inner_prod(x, &x[-T1], &x[-T1b], N, &xy, &xy2, arch); xy += xy2; yy = yy_lookup[T1] + yy_lookup[T1b]; #ifdef FIXED_POINT diff --git a/celt/pitch.h b/celt/pitch.h index 4368cc5..af745eb 100644 --- a/celt/pitch.h +++ b/celt/pitch.h @@ -37,8 +37,8 @@ #include "modes.h" #include "cpu_support.h" -#if defined(__SSE__) && !defined(FIXED_POINT) \ - || defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2) +#if (defined(OPUS_X86_MAY_HAVE_SSE) && !defined(FIXED_POINT)) \ + || ((defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2)) && defined(FIXED_POINT)) #include "x86/pitch_sse.h" #endif @@ -135,8 +135,7 @@ static OPUS_INLINE void xcorr_kernel_c(const opus_val16 * x, const opus_val16 * #endif /* OVERRIDE_XCORR_KERNEL */ -#ifndef OVERRIDE_DUAL_INNER_PROD -static OPUS_INLINE void dual_inner_prod(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, +static OPUS_INLINE void dual_inner_prod_c(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, int N, opus_val32 *xy1, opus_val32 *xy2) { int i; @@ -150,6 +149,10 @@ static OPUS_INLINE void dual_inner_prod(const opus_val16 *x, const opus_val16 *y *xy1 = xy01; *xy2 = xy02; } + +#ifndef OVERRIDE_DUAL_INNER_PROD +# define dual_inner_prod(x, y01, y02, N, xy1, xy2, arch) \ + ((void)(arch),dual_inner_prod_c(x, y01, y02, N, xy1, xy2)) #endif /*We make sure a C version is always available for cases where the overhead of @@ -169,6 +172,12 @@ static OPUS_INLINE opus_val32 celt_inner_prod_c(const opus_val16 *x, ((void)(arch),celt_inner_prod_c(x, y, N)) #endif +#ifdef NON_STATIC_COMB_FILTER_CONST_C +void comb_filter_const_c(opus_val32 *y, opus_val32 *x, int T, int N, + opus_val16 g10, opus_val16 g11, opus_val16 g12); +#endif + + #ifdef FIXED_POINT opus_val32 #else @@ -180,7 +189,7 @@ celt_pitch_xcorr_c(const opus_val16 *_x, const opus_val16 *_y, #if !defined(OVERRIDE_PITCH_XCORR) /*Is run-time CPU detection enabled on this platform?*/ # if defined(OPUS_HAVE_RTCD) && \ - (defined(OPUS_ARM_ASM) || defined(OPUS_ARM_NEON_INTR)) + (defined(OPUS_ARM_ASM) || (defined(OPUS_ARM_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR))) extern # if defined(FIXED_POINT) opus_val32 diff --git a/celt/tests/test_unit_mathops.c b/celt/tests/test_unit_mathops.c index b9b1bcf..2de39ba 100644 --- a/celt/tests/test_unit_mathops.c +++ b/celt/tests/test_unit_mathops.c @@ -49,10 +49,17 @@ #include "cwrs.c" #include "pitch.c" #include "celt_lpc.c" +#include "celt.c" -#if defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2) +#if defined(OPUS_X86_MAY_HAVE_SSE) || defined(OPUS_X86_MAY_HAVE_SSE2) || defined(OPUS_X86_MAY_HAVE_SSE4_1) +#if defined(OPUS_X86_MAY_HAVE_SSE) #include "x86/pitch_sse.c" +#endif +#if defined(OPUS_X86_MAY_HAVE_SSE2) +#include "x86/pitch_sse2.c" +#endif #if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#include "x86/pitch_sse4_1.c" #include "x86/celt_lpc_sse.c" #endif #include "x86/x86_celt_map.c" diff --git a/celt/tests/test_unit_rotation.c b/celt/tests/test_unit_rotation.c index 5507884..4780005 100644 --- a/celt/tests/test_unit_rotation.c +++ b/celt/tests/test_unit_rotation.c @@ -46,11 +46,18 @@ #include "bands.h" #include "pitch.c" #include "celt_lpc.c" +#include "celt.c" #include -#if defined(OPUS_X86_MAY_HAVE_SSE4_1) || defined(OPUS_X86_MAY_HAVE_SSE2) +#if defined(OPUS_X86_MAY_HAVE_SSE) || defined(OPUS_X86_MAY_HAVE_SSE2) || defined(OPUS_X86_MAY_HAVE_SSE4_1) +#if defined(OPUS_X86_MAY_HAVE_SSE) #include "x86/pitch_sse.c" +#endif +#if defined(OPUS_X86_MAY_HAVE_SSE2) +#include "x86/pitch_sse2.c" +#endif #if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#include "x86/pitch_sse4_1.c" #include "x86/celt_lpc_sse.c" #endif #include "x86/x86_celt_map.c" diff --git a/celt/x86/celt_lpc_sse.c b/celt/x86/celt_lpc_sse.c index 9fb9779..67e5592 100644 --- a/celt/x86/celt_lpc_sse.c +++ b/celt/x86/celt_lpc_sse.c @@ -38,6 +38,8 @@ #include "pitch.h" #include "x86cpu.h" +#if defined(FIXED_POINT) + void celt_fir_sse4_1(const opus_val16 *_x, const opus_val16 *num, opus_val16 *_y, @@ -126,3 +128,5 @@ void celt_fir_sse4_1(const opus_val16 *_x, #endif RESTORE_STACK; } + +#endif diff --git a/celt/x86/celt_lpc_sse.h b/celt/x86/celt_lpc_sse.h index f111420..c5ec796 100644 --- a/celt/x86/celt_lpc_sse.h +++ b/celt/x86/celt_lpc_sse.h @@ -32,7 +32,9 @@ #include "config.h" #endif -#if defined(OPUS_X86_MAY_HAVE_SSE4_1) +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) && defined(FIXED_POINT) +#define OVERRIDE_CELT_FIR + void celt_fir_sse4_1( const opus_val16 *x, const opus_val16 *num, @@ -42,6 +44,12 @@ void celt_fir_sse4_1( opus_val16 *mem, int arch); +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define celt_fir(x, num, y, N, ord, mem, arch) \ + ((void)arch, celt_fir_sse4_1(x, num, y, N, ord, mem, arch)) + +#else + extern void (*const CELT_FIR_IMPL[OPUS_ARCHMASK + 1])( const opus_val16 *x, const opus_val16 *num, @@ -56,3 +64,5 @@ extern void (*const CELT_FIR_IMPL[OPUS_ARCHMASK + 1])( #endif #endif + +#endif diff --git a/celt/x86/pitch_sse.c b/celt/x86/pitch_sse.c index e3bc6d7..20e7312 100644 --- a/celt/x86/pitch_sse.c +++ b/celt/x86/pitch_sse.c @@ -29,223 +29,157 @@ #include "config.h" #endif -#include -#include - #include "macros.h" #include "celt_lpc.h" #include "stack_alloc.h" #include "mathops.h" #include "pitch.h" -#if defined(OPUS_X86_MAY_HAVE_SSE4_1) -#include -#include "x86cpu.h" - -opus_val32 celt_inner_prod_sse4_1(const opus_val16 *x, const opus_val16 *y, - int N) -{ - opus_int i, dataSize16; - opus_int32 sum; - __m128i inVec1_76543210, inVec1_FEDCBA98, acc1; - __m128i inVec2_76543210, inVec2_FEDCBA98, acc2; - __m128i inVec1_3210, inVec2_3210; - - sum = 0; - dataSize16 = N & ~15; - - acc1 = _mm_setzero_si128(); - acc2 = _mm_setzero_si128(); - - for (i=0;i= 8) - { - inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); - inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); - - inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); - - acc1 = _mm_add_epi32(acc1, inVec1_76543210); - i += 8; - } - - if (N - i >= 4) - { - inVec1_3210 = OP_CVTEPI16_EPI32_M64(&x[i + 0]); - inVec2_3210 = OP_CVTEPI16_EPI32_M64(&y[i + 0]); - - inVec1_3210 = _mm_mullo_epi32(inVec1_3210, inVec2_3210); - - acc1 = _mm_add_epi32(acc1, inVec1_3210); - i += 4; - } - - acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64(acc1, acc1)); - acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16(acc1, 0x0E)); - - sum += _mm_cvtsi128_si32(acc1); - - for (;i +#include "arch.h" - return sum; +void xcorr_kernel_sse(const opus_val16 *x, const opus_val16 *y, opus_val32 sum[4], int len) +{ + int j; + __m128 xsum1, xsum2; + xsum1 = _mm_loadu_ps(sum); + xsum2 = _mm_setzero_ps(); + + for (j = 0; j < len-3; j += 4) + { + __m128 x0 = _mm_loadu_ps(x+j); + __m128 yj = _mm_loadu_ps(y+j); + __m128 y3 = _mm_loadu_ps(y+j+3); + + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x00),yj)); + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x55), + _mm_shuffle_ps(yj,y3,0x49))); + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xaa), + _mm_shuffle_ps(yj,y3,0x9e))); + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xff),y3)); + } + if (j < len) + { + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + if (++j < len) + { + xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + if (++j < len) + { + xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); + } + } + } + _mm_storeu_ps(sum,_mm_add_ps(xsum1,xsum2)); } -void xcorr_kernel_sse4_1(const opus_val16 * x, const opus_val16 * y, opus_val32 sum[ 4 ], int len) + +void dual_inner_prod_sse(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, + int N, opus_val32 *xy1, opus_val32 *xy2) { - int j; - - __m128i vecX, vecX0, vecX1, vecX2, vecX3; - __m128i vecY0, vecY1, vecY2, vecY3; - __m128i sum0, sum1, sum2, sum3, vecSum; - __m128i initSum; - - celt_assert(len >= 3); - - sum0 = _mm_setzero_si128(); - sum1 = _mm_setzero_si128(); - sum2 = _mm_setzero_si128(); - sum3 = _mm_setzero_si128(); - - for (j=0;j<(len-7);j+=8) - { - vecX = _mm_loadu_si128((__m128i *)(&x[j + 0])); - vecY0 = _mm_loadu_si128((__m128i *)(&y[j + 0])); - vecY1 = _mm_loadu_si128((__m128i *)(&y[j + 1])); - vecY2 = _mm_loadu_si128((__m128i *)(&y[j + 2])); - vecY3 = _mm_loadu_si128((__m128i *)(&y[j + 3])); - - sum0 = _mm_add_epi32(sum0, _mm_madd_epi16(vecX, vecY0)); - sum1 = _mm_add_epi32(sum1, _mm_madd_epi16(vecX, vecY1)); - sum2 = _mm_add_epi32(sum2, _mm_madd_epi16(vecX, vecY2)); - sum3 = _mm_add_epi32(sum3, _mm_madd_epi16(vecX, vecY3)); - } - - sum0 = _mm_add_epi32(sum0, _mm_unpackhi_epi64( sum0, sum0)); - sum0 = _mm_add_epi32(sum0, _mm_shufflelo_epi16( sum0, 0x0E)); - - sum1 = _mm_add_epi32(sum1, _mm_unpackhi_epi64( sum1, sum1)); - sum1 = _mm_add_epi32(sum1, _mm_shufflelo_epi16( sum1, 0x0E)); - - sum2 = _mm_add_epi32(sum2, _mm_unpackhi_epi64( sum2, sum2)); - sum2 = _mm_add_epi32(sum2, _mm_shufflelo_epi16( sum2, 0x0E)); - - sum3 = _mm_add_epi32(sum3, _mm_unpackhi_epi64( sum3, sum3)); - sum3 = _mm_add_epi32(sum3, _mm_shufflelo_epi16( sum3, 0x0E)); - - vecSum = _mm_unpacklo_epi64(_mm_unpacklo_epi32(sum0, sum1), - _mm_unpacklo_epi32(sum2, sum3)); - - for (;j<(len-3);j+=4) - { - vecX = OP_CVTEPI16_EPI32_M64(&x[j + 0]); - vecX0 = _mm_shuffle_epi32(vecX, 0x00); - vecX1 = _mm_shuffle_epi32(vecX, 0x55); - vecX2 = _mm_shuffle_epi32(vecX, 0xaa); - vecX3 = _mm_shuffle_epi32(vecX, 0xff); - - vecY0 = OP_CVTEPI16_EPI32_M64(&y[j + 0]); - vecY1 = OP_CVTEPI16_EPI32_M64(&y[j + 1]); - vecY2 = OP_CVTEPI16_EPI32_M64(&y[j + 2]); - vecY3 = OP_CVTEPI16_EPI32_M64(&y[j + 3]); - - sum0 = _mm_mullo_epi32(vecX0, vecY0); - sum1 = _mm_mullo_epi32(vecX1, vecY1); - sum2 = _mm_mullo_epi32(vecX2, vecY2); - sum3 = _mm_mullo_epi32(vecX3, vecY3); - - sum0 = _mm_add_epi32(sum0, sum1); - sum2 = _mm_add_epi32(sum2, sum3); - vecSum = _mm_add_epi32(vecSum, sum0); - vecSum = _mm_add_epi32(vecSum, sum2); - } - - for (;j= 8) - { - inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); - inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); - - inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); + int i; + float xy; + __m128 sum; + sum = _mm_setzero_ps(); + /* FIXME: We should probably go 8-way and use 2 sums. */ + for (i=0;i -#include "arch.h" +#endif -#define OVERRIDE_XCORR_KERNEL -static OPUS_INLINE void xcorr_kernel_sse(const opus_val16 *x, const opus_val16 *y, opus_val32 sum[4], int len) -{ - int j; - __m128 xsum1, xsum2; - xsum1 = _mm_loadu_ps(sum); - xsum2 = _mm_setzero_ps(); - - for (j = 0; j < len-3; j += 4) - { - __m128 x0 = _mm_loadu_ps(x+j); - __m128 yj = _mm_loadu_ps(y+j); - __m128 y3 = _mm_loadu_ps(y+j+3); - - xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x00),yj)); - xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0x55), - _mm_shuffle_ps(yj,y3,0x49))); - xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xaa), - _mm_shuffle_ps(yj,y3,0x9e))); - xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_shuffle_ps(x0,x0,0xff),y3)); - } - if (j < len) - { - xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); - if (++j < len) - { - xsum2 = _mm_add_ps(xsum2,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); - if (++j < len) - { - xsum1 = _mm_add_ps(xsum1,_mm_mul_ps(_mm_load1_ps(x+j),_mm_loadu_ps(y+j))); - } - } - } - _mm_storeu_ps(sum,_mm_add_ps(xsum1,xsum2)); -} - -#define xcorr_kernel(_x, _y, _z, len, arch) \ - ((void)(arch),xcorr_kernel_sse(_x, _y, _z, len)) +#if defined(OPUS_X86_MAY_HAVE_SSE) && !defined(FIXED_POINT) #define OVERRIDE_DUAL_INNER_PROD -static OPUS_INLINE void dual_inner_prod(const opus_val16 *x, const opus_val16 *y01, const opus_val16 *y02, - int N, opus_val32 *xy1, opus_val32 *xy2) -{ - int i; - __m128 xsum1, xsum2; - xsum1 = _mm_setzero_ps(); - xsum2 = _mm_setzero_ps(); - for (i=0;i +#include + +#include "macros.h" +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE2) && defined(FIXED_POINT) +opus_val32 celt_inner_prod_sse2(const opus_val16 *x, const opus_val16 *y, + int N) +{ + opus_int i, dataSize16; + opus_int32 sum; + + __m128i inVec1_76543210, inVec1_FEDCBA98, acc1; + __m128i inVec2_76543210, inVec2_FEDCBA98, acc2; + + sum = 0; + dataSize16 = N & ~15; + + acc1 = _mm_setzero_si128(); + acc2 = _mm_setzero_si128(); + + for (i=0;i= 8) + { + inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); + inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); + + inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); + + acc1 = _mm_add_epi32(acc1, inVec1_76543210); + i += 8; + } + + acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64( acc1, acc1)); + acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16( acc1, 0x0E)); + sum += _mm_cvtsi128_si32(acc1); + + for (;i +#include + +#include "macros.h" +#include "celt_lpc.h" +#include "stack_alloc.h" +#include "mathops.h" +#include "pitch.h" + +#if defined(OPUS_X86_MAY_HAVE_SSE4_1) && defined(FIXED_POINT) +#include +#include "x86cpu.h" + +opus_val32 celt_inner_prod_sse4_1(const opus_val16 *x, const opus_val16 *y, + int N) +{ + opus_int i, dataSize16; + opus_int32 sum; + __m128i inVec1_76543210, inVec1_FEDCBA98, acc1; + __m128i inVec2_76543210, inVec2_FEDCBA98, acc2; + __m128i inVec1_3210, inVec2_3210; + + sum = 0; + dataSize16 = N & ~15; + + acc1 = _mm_setzero_si128(); + acc2 = _mm_setzero_si128(); + + for (i=0;i= 8) + { + inVec1_76543210 = _mm_loadu_si128((__m128i *)(&x[i + 0])); + inVec2_76543210 = _mm_loadu_si128((__m128i *)(&y[i + 0])); + + inVec1_76543210 = _mm_madd_epi16(inVec1_76543210, inVec2_76543210); + + acc1 = _mm_add_epi32(acc1, inVec1_76543210); + i += 8; + } + + if (N - i >= 4) + { + inVec1_3210 = OP_CVTEPI16_EPI32_M64(&x[i + 0]); + inVec2_3210 = OP_CVTEPI16_EPI32_M64(&y[i + 0]); + + inVec1_3210 = _mm_mullo_epi32(inVec1_3210, inVec2_3210); + + acc1 = _mm_add_epi32(acc1, inVec1_3210); + i += 4; + } + + acc1 = _mm_add_epi32(acc1, _mm_unpackhi_epi64(acc1, acc1)); + acc1 = _mm_add_epi32(acc1, _mm_shufflelo_epi16(acc1, 0x0E)); + + sum += _mm_cvtsi128_si32(acc1); + + for (;i= 3); + + sum0 = _mm_setzero_si128(); + sum1 = _mm_setzero_si128(); + sum2 = _mm_setzero_si128(); + sum3 = _mm_setzero_si128(); + + for (j=0;j<(len-7);j+=8) + { + vecX = _mm_loadu_si128((__m128i *)(&x[j + 0])); + vecY0 = _mm_loadu_si128((__m128i *)(&y[j + 0])); + vecY1 = _mm_loadu_si128((__m128i *)(&y[j + 1])); + vecY2 = _mm_loadu_si128((__m128i *)(&y[j + 2])); + vecY3 = _mm_loadu_si128((__m128i *)(&y[j + 3])); + + sum0 = _mm_add_epi32(sum0, _mm_madd_epi16(vecX, vecY0)); + sum1 = _mm_add_epi32(sum1, _mm_madd_epi16(vecX, vecY1)); + sum2 = _mm_add_epi32(sum2, _mm_madd_epi16(vecX, vecY2)); + sum3 = _mm_add_epi32(sum3, _mm_madd_epi16(vecX, vecY3)); + } + + sum0 = _mm_add_epi32(sum0, _mm_unpackhi_epi64( sum0, sum0)); + sum0 = _mm_add_epi32(sum0, _mm_shufflelo_epi16( sum0, 0x0E)); + + sum1 = _mm_add_epi32(sum1, _mm_unpackhi_epi64( sum1, sum1)); + sum1 = _mm_add_epi32(sum1, _mm_shufflelo_epi16( sum1, 0x0E)); + + sum2 = _mm_add_epi32(sum2, _mm_unpackhi_epi64( sum2, sum2)); + sum2 = _mm_add_epi32(sum2, _mm_shufflelo_epi16( sum2, 0x0E)); + + sum3 = _mm_add_epi32(sum3, _mm_unpackhi_epi64( sum3, sum3)); + sum3 = _mm_add_epi32(sum3, _mm_shufflelo_epi16( sum3, 0x0E)); + + vecSum = _mm_unpacklo_epi64(_mm_unpacklo_epi32(sum0, sum1), + _mm_unpacklo_epi32(sum2, sum3)); + + for (;j<(len-3);j+=4) + { + vecX = OP_CVTEPI16_EPI32_M64(&x[j + 0]); + vecX0 = _mm_shuffle_epi32(vecX, 0x00); + vecX1 = _mm_shuffle_epi32(vecX, 0x55); + vecX2 = _mm_shuffle_epi32(vecX, 0xaa); + vecX3 = _mm_shuffle_epi32(vecX, 0xff); + + vecY0 = OP_CVTEPI16_EPI32_M64(&y[j + 0]); + vecY1 = OP_CVTEPI16_EPI32_M64(&y[j + 1]); + vecY2 = OP_CVTEPI16_EPI32_M64(&y[j + 2]); + vecY3 = OP_CVTEPI16_EPI32_M64(&y[j + 3]); + + sum0 = _mm_mullo_epi32(vecX0, vecY0); + sum1 = _mm_mullo_epi32(vecX1, vecY1); + sum2 = _mm_mullo_epi32(vecX2, vecY2); + sum3 = _mm_mullo_epi32(vecX3, vecY3); + + sum0 = _mm_add_epi32(sum0, sum1); + sum2 = _mm_add_epi32(sum2, sum3); + vecSum = _mm_add_epi32(vecSum, sum0); + vecSum = _mm_add_epi32(vecSum, sum2); + } + + for (;j -#define cpuid(info,x) __cpuid(info,x) +static _inline void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) +{ + __cpuid((int*)CPUInfo, InfoType); +} + #else #if defined(CPU_INFO_BY_C) @@ -48,14 +57,28 @@ static void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) { #if defined(CPU_INFO_BY_ASM) +#if defined(__i386__) && defined(__PIC__) +/* %ebx is PIC register in 32-bit, so mustn't clobber it. */ + __asm__ __volatile__ ( + "xchg %%ebx, %1\n" + "cpuid\n" + "xchg %%ebx, %1\n": + "=a" (CPUInfo[0]), + "=r" (CPUInfo[1]), + "=c" (CPUInfo[2]), + "=d" (CPUInfo[3]) : + "0" (InfoType) + ); +#else __asm__ __volatile__ ( "cpuid": "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : - "a" (InfoType), "c" (0) + "0" (InfoType) ); +#endif #elif defined(CPU_INFO_BY_C) __get_cpuid(InfoType, &(CPUInfo[0]), &(CPUInfo[1]), &(CPUInfo[2]), &(CPUInfo[3])); #endif @@ -63,11 +86,9 @@ static void cpuid(unsigned int CPUInfo[4], unsigned int InfoType) #endif -#include "SigProc_FIX.h" -#include "celt_lpc.h" - typedef struct CPU_Feature{ /* SIMD: 128-bit */ + int HW_SSE; int HW_SSE2; int HW_SSE41; } CPU_Feature; @@ -82,19 +103,31 @@ static void opus_cpu_feature_check(CPU_Feature *cpu_feature) if (nIds >= 1){ cpuid(info, 1); + cpu_feature->HW_SSE = (info[3] & (1 << 25)) != 0; cpu_feature->HW_SSE2 = (info[3] & (1 << 26)) != 0; cpu_feature->HW_SSE41 = (info[2] & (1 << 19)) != 0; } + else { + cpu_feature->HW_SSE = 0; + cpu_feature->HW_SSE2 = 0; + cpu_feature->HW_SSE41 = 0; + } } int opus_select_arch(void) { - CPU_Feature cpu_feature = {0}; + CPU_Feature cpu_feature; int arch; opus_cpu_feature_check(&cpu_feature); arch = 0; + if (!cpu_feature.HW_SSE) + { + return arch; + } + arch++; + if (!cpu_feature.HW_SSE2) { return arch; @@ -109,3 +142,5 @@ int opus_select_arch(void) return arch; } + +#endif diff --git a/celt/x86/x86cpu.h b/celt/x86/x86cpu.h index ef53f0c..7f4c61d 100644 --- a/celt/x86/x86cpu.h +++ b/celt/x86/x86cpu.h @@ -28,6 +28,12 @@ #if !defined(X86CPU_H) # define X86CPU_H +# if defined(OPUS_X86_MAY_HAVE_SSE) +# define MAY_HAVE_SSE(name) name ## _sse +# else +# define MAY_HAVE_SSE(name) name ## _c +# endif + # if defined(OPUS_X86_MAY_HAVE_SSE2) # define MAY_HAVE_SSE2(name) name ## _sse2 # else @@ -55,21 +61,25 @@ int opus_select_arch(void); reference in the PMOVSXWD instruction itself, but gcc is not smart enough to optimize this out when optimizations ARE enabled. - It appears clang requires us to do this always (which is fair, since - technically the compiler is always allowed to do the dereference before - invoking the function implementing the intrinsic). I have not investiaged - whether it is any smarter than gcc when it comes to eliminating the extra - load instruction.*/ + Clang, in contrast, requires us to do this always for _mm_cvtepi8_epi32 + (which is fair, since technically the compiler is always allowed to do the + dereference before invoking the function implementing the intrinsic). + However, it is smart enough to eliminate the extra MOVD instruction. + For _mm_cvtepi16_epi32, it does the right thing, though does *not* optimize out + the extra MOVQ if it's specified explicitly */ + # if defined(__clang__) || !defined(__OPTIMIZE__) # define OP_CVTEPI8_EPI32_M32(x) \ (_mm_cvtepi8_epi32(_mm_cvtsi32_si128(*(int *)(x)))) - -# define OP_CVTEPI16_EPI32_M64(x) \ - (_mm_cvtepi16_epi32(_mm_loadl_epi64((__m128i *)(x)))) # else # define OP_CVTEPI8_EPI32_M32(x) \ (_mm_cvtepi8_epi32(*(__m128i *)(x))) +#endif +# if !defined(__OPTIMIZE__) +# define OP_CVTEPI16_EPI32_M64(x) \ + (_mm_cvtepi16_epi32(_mm_loadl_epi64((__m128i *)(x)))) +# else # define OP_CVTEPI16_EPI32_M64(x) \ (_mm_cvtepi16_epi32(*(__m128i *)(x))) # endif diff --git a/celt_sources.mk b/celt_sources.mk index 29ec937..c92693f 100644 --- a/celt_sources.mk +++ b/celt_sources.mk @@ -21,7 +21,10 @@ CELT_SOURCES_SSE = celt/x86/x86cpu.c \ celt/x86/x86_celt_map.c \ celt/x86/pitch_sse.c -CELT_SOURCES_SSE4_1 = celt/x86/celt_lpc_sse.c +CELT_SOURCES_SSE2 = celt/x86/pitch_sse2.c + +CELT_SOURCES_SSE4_1 = celt/x86/celt_lpc_sse.c \ +celt/x86/pitch_sse4_1.c CELT_SOURCES_ARM = \ celt/arm/armcpu.c \ diff --git a/configure.ac b/configure.ac index 87cece9..b463496 100644 --- a/configure.ac +++ b/configure.ac @@ -348,64 +348,189 @@ AM_CONDITIONAL([OPUS_ARM_INLINE_ASM], AM_CONDITIONAL([OPUS_ARM_EXTERNAL_ASM], [test x"${asm_optimization%% *}" = x"ARM"]) -AM_CONDITIONAL([HAVE_SSE4_1], [false]) +AM_CONDITIONAL([HAVE_SSE], [false]) AM_CONDITIONAL([HAVE_SSE2], [false]) +AM_CONDITIONAL([HAVE_SSE4_1], [false]) + +m4_define([DEFAULT_X86_SSE_CFLAGS], [-msse]) +m4_define([DEFAULT_X86_SSE2_CFLAGS], [-msse2]) +m4_define([DEFAULT_X86_SSE4_1_CFLAGS], [-msse4.1]) +m4_define([DEFAULT_ARM_NEON_INTR_CFLAGS], [-mfpu=neon]) + +AC_ARG_VAR([X86_SSE_CFLAGS], [C compiler flags to compile SSE intrinsics @<:@default=]DEFAULT_X86_SSE_CFLAGS[@:>@]) +AC_ARG_VAR([X86_SSE2_CFLAGS], [C compiler flags to compile SSE2 intrinsics @<:@default=]DEFAULT_X86_SSE2_CFLAGS[@:>@]) +AC_ARG_VAR([X86_SSE4_1_CFLAGS], [C compiler flags to compile SSE4.1 intrinsics @<:@default=]DEFAULT_X86_SSE4_1_CFLAGS[@:>@]) +AC_ARG_VAR([ARM_NEON_INTR_CFLAGS], [C compiler flags to compile ARM NEON intrinsics @<:@default=]DEFAULT_ARM_NEON_INTR_CFLAGS[@:>@]) + +AS_VAR_SET_IF([X86_SSE_CFLAGS], [], [AS_VAR_SET([X86_SSE_CFLAGS], DEFAULT_X86_SSE_CFLAGS)]) +AS_VAR_SET_IF([X86_SSE2_CFLAGS], [], [AS_VAR_SET([X86_SSE2_CFLAGS], DEFAULT_X86_SSE2_CFLAGS)]) +AS_VAR_SET_IF([X86_SSE4_1_CFLAGS], [], [AS_VAR_SET([X86_SSE4_1_CFLAGS], DEFAULT_X86_SSE4_1_CFLAGS)]) +AS_VAR_SET_IF([ARM_NEON_INTR_CFLAGS], [], [AS_VAR_SET([ARM_NEON_INTR_CFLAGS], DEFAULT_ARM_NEON_INTR_CFLAGS)]) AS_IF([test x"$enable_intrinsics" = x"yes"],[ - case $host_cpu in - arm*) + intrinsics_support="" + AS_CASE([$host_cpu], + [arm*], + [ cpu_arm=yes - AC_MSG_CHECKING(if compiler supports ARM NEON intrinsics) - save_CFLAGS="$CFLAGS"; CFLAGS="-mfpu=neon $CFLAGS" - AC_LINK_IFELSE( - [ - AC_LANG_PROGRAM( - [[#include - ]], - [[ - static float32x4_t A[2], SUMM; - SUMM = vmlaq_f32(SUMM, A[0], A[1]); - ]] - ) - ],[ - OPUS_ARM_NEON_INTR=1 - AC_MSG_RESULT([yes]) - ],[ - OPUS_ARM_NEON_INTR=0 - AC_MSG_RESULT([no]) - ] + OPUS_CHECK_INTRINSICS( + [ARM Neon], + [$ARM_NEON_INTR_CFLAGS], + [OPUS_ARM_MAY_HAVE_NEON_INTR], + [OPUS_ARM_PRESUME_NEON_INTR], + [[#include + ]], + [[ + static float32x4_t A0, A1, SUMM; + SUMM = vmlaq_f32(SUMM, A0, A1); + ]] + ) + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1" && test x"$OPUS_ARM_PRESUME_NEON_INTR" != x"1"], + [ + OPUS_ARM_NEON_INTR_CFLAGS="$ARM_NEON_INTR_CFLAGS" + AC_SUBST([OPUS_ARM_NEON_INTR_CFLAGS]) + ] ) - CFLAGS="$save_CFLAGS" - #Now we know if compiler supports ARM neon intrinsics or not - #Currently we only have intrinsic optimization for floating point + #Currently we only have intrinsic optimizations for floating point AS_IF([test x"$enable_float" = x"yes"], [ - AS_IF([test x"$OPUS_ARM_NEON_INTR" = x"1"], + AS_IF([test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1"], [ - AC_DEFINE([OPUS_ARM_NEON_INTR], 1, [Compiler supports ARMv7 Neon Intrinsics]) - AS_IF([test x"enable_rtcd" != x""], - [rtcd_support="ARM (ARMv7_Neon_Intrinsics)"],[]) - enable_intrinsics="$enable_intrinsics ARMv7_Neon_Intrinsics" - dnl Don't see why defining these is necessary to check features at runtime - AC_DEFINE([OPUS_ARM_MAY_HAVE_EDSP], 1, [Define if compiler support EDSP Instructions]) - AC_DEFINE([OPUS_ARM_MAY_HAVE_MEDIA], 1, [Define if compiler support MEDIA Instructions]) - AC_DEFINE([OPUS_ARM_MAY_HAVE_NEON], 1, [Define if compiler support NEON instructions]) + AC_DEFINE([OPUS_ARM_MAY_HAVE_NEON_INTR], 1, [Compiler supports ARMv7 Neon Intrinsics]) + intrinsics_support="$intrinsics_support (Neon_Intrinsics)" + + AS_IF([test x"enable_rtcd" != x"" && test x"$OPUS_ARM_PRESUME_NEON_INTR" != x"1"], + [rtcd_support="$rtcd_support (ARMv7_Neon_Intrinsics)"],[]) + + AS_IF([test x"$OPUS_ARM_PRESUME_NEON_INTR" = x"1"], + [AC_DEFINE([OPUS_ARM_PRESUME_NEON_INTR], 1, [Define if binary requires NEON intrinsics support])]) + + AS_IF([test x"$rtcd_support" = x""], + [rtcd_support=no]) + + AS_IF([test x"$intrinsics_support" = x""], + [intrinsics_support=no], + [intrinsics_support="arm$intrinsics_support"]) ], [ AC_MSG_WARN([Compiler does not support ARM intrinsics]) - enable_intrinsics=no + intrinsics_support=no ]) ], [ - AC_MSG_WARN([Currently on have ARM intrinsics for float]) - enable_intrinsics=no + AC_MSG_WARN([Currently only have ARM intrinsics for float]) + intrinsics_support=no ]) - ;; - "i386" | "i686" | "x86_64") - AS_IF([test x"$enable_float" = x"no"],[ - AS_IF([test x"$enable_rtcd" = x"yes"],[ + ], + [i?86|x86_64], + [ + OPUS_CHECK_INTRINSICS( + [SSE], + [$X86_SSE_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE], + [OPUS_X86_PRESUME_SSE], + [[#include + ]], + [[ + static __m128 mtest; + mtest = _mm_setzero_ps(); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE" = x"1" && test x"$OPUS_X86_PRESUME_SSE" != x"1"], + [ + OPUS_X86_SSE_CFLAGS="$X86_SSE_CFLAGS" + AC_SUBST([OPUS_X86_SSE_CFLAGS]) + ] + ) + OPUS_CHECK_INTRINSICS( + [SSE2], + [$X86_SSE2_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE2], + [OPUS_X86_PRESUME_SSE2], + [[#include + ]], + [[ + static __m128i mtest; + mtest = _mm_setzero_si128(); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1" && test x"$OPUS_X86_PRESUME_SSE2" != x"1"], + [ + OPUS_X86_SSE2_CFLAGS="$X86_SSE2_CFLAGS" + AC_SUBST([OPUS_X86_SSE2_CFLAGS]) + ] + ) + OPUS_CHECK_INTRINSICS( + [SSE4.1], + [$X86_SSE4_1_CFLAGS], + [OPUS_X86_MAY_HAVE_SSE4_1], + [OPUS_X86_PRESUME_SSE4_1], + [[#include + ]], + [[ + static __m128i mtest; + mtest = _mm_setzero_si128(); + mtest = _mm_cmpeq_epi64(mtest, mtest); + ]] + ) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1" && test x"$OPUS_X86_PRESUME_SSE4_1" != x"1"], + [ + OPUS_X86_SSE4_1_CFLAGS="$X86_SSE4_1_CFLAGS" + AC_SUBST([OPUS_X86_SSE4_1_CFLAGS]) + ] + ) + + AS_IF([test x"$rtcd_support" = x"no"], [rtcd_support=""]) + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE], 1, [Compiler supports X86 SSE Intrinsics]) + intrinsics_support="$intrinsics_support SSE" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE], 1, [Define if binary requires SSE intrinsics support])], + [rtcd_support="$rtcd_support SSE"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE intrinsics]) + ]) + + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE2], 1, [Compiler supports X86 SSE2 Intrinsics]) + intrinsics_support="$intrinsics_support SSE2" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE2" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE2], 1, [Define if binary requires SSE2 intrinsics support])], + [rtcd_support="$rtcd_support SSE2"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE2 intrinsics]) + ]) + + AS_IF([test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1"], + [ + AC_DEFINE([OPUS_X86_MAY_HAVE_SSE4_1], 1, [Compiler supports X86 SSE4.1 Intrinsics]) + intrinsics_support="$intrinsics_support SSE4.1" + + AS_IF([test x"$OPUS_X86_PRESUME_SSE4_1" = x"1"], + [AC_DEFINE([OPUS_X86_PRESUME_SSE4_1], 1, [Define if binary requires SSE4.1 intrinsics support])], + [rtcd_support="$rtcd_support SSE4.1"]) + ], + [ + AC_MSG_WARN([Compiler does not support SSE4.1 intrinsics]) + ]) + AS_IF([test x"$intrinsics_support" = x""], + [intrinsics_support=no], + [intrinsics_support="x86$intrinsics_support"] + ) + AS_IF([test x"$rtcd_support" = x""], + [rtcd_support=no], + [rtcd_support="x86$rtcd_support"], + ) + + AS_IF([test x"$enable_rtcd" = x"yes" && test x"$rtcd_support" != x""],[ get_cpuid_by_asm="no" - AC_MSG_CHECKING([Get CPU Info]) + AC_MSG_CHECKING([How to get X86 CPU Info]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include ]],[[ @@ -415,7 +540,7 @@ AS_IF([test x"$enable_intrinsics" = x"yes"],[ unsigned int CPUInfo3; unsigned int InfoType; __asm__ __volatile__ ( - "cpuid11": + "cpuid": "=a" (CPUInfo0), "=b" (CPUInfo1), "=c" (CPUInfo2), @@ -424,7 +549,8 @@ AS_IF([test x"$enable_intrinsics" = x"yes"],[ ); ]])], [get_cpuid_by_asm="yes" - AC_MSG_RESULT([Inline Assembly])], + AC_MSG_RESULT([Inline Assembly]) + AC_DEFINE([CPU_INFO_BY_ASM], [1], [Get CPU Info by asm method])], [AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include ]],[[ @@ -435,87 +561,28 @@ AS_IF([test x"$enable_intrinsics" = x"yes"],[ unsigned int InfoType; __get_cpuid(InfoType, &CPUInfo0, &CPUInfo1, &CPUInfo2, &CPUInfo3); ]])], - [AC_MSG_RESULT([C method])], - [AC_MSG_ERROR([not support Get CPU Info, please disable intrinsics ])])]) - - AC_MSG_CHECKING([sse4.1]) - TMP_CFLAGS="$CFLAGS" - gcc -Q --help=target | grep "\-msse4.1 " - AS_IF([test x"$?" = x"0"],[ - CFLAGS="$CFLAGS -msse4.1" - AC_CHECK_HEADER(xmmintrin.h, [], [AC_MSG_ERROR([Couldn't find xmmintrin.h])]) - AC_CHECK_HEADER(emmintrin.h, [], [AC_MSG_ERROR([Couldn't find emmintrin.h])]) - AC_CHECK_HEADER(smmintrin.h, [], [AC_MSG_ERROR([Couldn't find smmintrin.h])],[ - #ifdef HAVE_XMMINSTRIN_H - #include - #endif - #ifdef HAVE_EMMINSTRIN_H - #include - #endif - ]) - - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #include - #include - #include - ]],[[ - __m128i mtest = _mm_setzero_si128(); - mtest = _mm_cmpeq_epi64(mtest, mtest); - ]])], - [AC_MSG_RESULT([yes])], [AC_MSG_ERROR([Compiler & linker failure for sse4.1, please disable intrinsics])]) - - CFLAGS="$TMP_CFLAGS" - AC_DEFINE([OPUS_X86_MAY_HAVE_SSE4_1], [1], [For x86 sse4.1 instrinsics optimizations]) - AC_DEFINE([OPUS_X86_MAY_HAVE_SSE2], [1], [For x86 sse2 instrinsics optimizations]) - rtcd_support="x86 sse4.1" - AM_CONDITIONAL([HAVE_SSE4_1], [true]) - AM_CONDITIONAL([HAVE_SSE2], [true]) - AS_IF([test x"$get_cpuid_by_asm" = x"yes"],[AC_DEFINE([CPU_INFO_BY_ASM], [1], [Get CPU Info by asm method])], - [AC_DEFINE([CPU_INFO_BY_C], [1], [Get CPU Info by C method])]) - ],[ ##### Else case for AS_IF([test x"$?" = x"0"]) - gcc -Q --help=target | grep "\-msse2 " - AC_MSG_CHECKING([sse2]) - AS_IF([test x"$?" = x"0"],[ - AC_MSG_RESULT([yes]) - CFLAGS="$CFLAGS -msse2" - AC_CHECK_HEADER(xmmintrin.h, [], [AC_MSG_ERROR([Couldn't find xmmintrin.h])]) - AC_CHECK_HEADER(emmintrin.h, [], [AC_MSG_ERROR([Couldn't find emmintrin.h])]) - - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - #include - #include - ]],[[ - __m128i mtest = _mm_setzero_si128(); - ]])], - [AC_MSG_RESULT([yes])], [AC_MSG_ERROR([Compiler & linker failure for sse2, please disable intrinsics])]) - - CFLAGS="$TMP_CFLAGS" - AC_DEFINE([OPUS_X86_MAY_HAVE_SSE2], [1], [For x86 sse2 instrinsics optimize]) - rtcd_support="x86 sse2" - AM_CONDITIONAL([HAVE_SSE2], [true]) - AS_IF([test x"$get_cpuid_by_asm" = x"yes"],[AC_DEFINE([CPU_INFO_BY_ASM], [1], [Get CPU Info by asm method])], - [AC_DEFINE([CPU_INFO_BY_C], [1], [Get CPU Info by c method])]) - ],[enable_intrinsics="no"]) #End of AS_IF([test x"$?" = x"0"] - ]) - ], [ - enable_intrinsics="no" - ]) ## End of AS_IF([test x"$enable_rtcd" = x"yes"] -], -[ ## Else case for AS_IF([test x"$enable_float" = x"no"] - AC_MSG_WARN([Disabling intrinsics .. x86 intrinsics only avail for fixed point]) - enable_intrinsics="no" -]) ## End of AS_IF([test x"$enable_float" = x"no"] - ;; - *) + [AC_MSG_RESULT([C method]) + AC_DEFINE([CPU_INFO_BY_C], [1], [Get CPU Info by c method])], + [AC_MSG_ERROR([no supported Get CPU Info method, please disable intrinsics])])])]) + ], + [ AC_MSG_WARN([No intrinsics support for your architecture]) - enable_intrinsics="no" - ;; - esac + intrinsics_support="no" + ]) +], +[ + intrinsics_support="no" ]) AM_CONDITIONAL([CPU_ARM], [test "$cpu_arm" = "yes"]) AM_CONDITIONAL([OPUS_ARM_NEON_INTR], - [test x"$OPUS_ARM_NEON_INTR" = x"1"]) + [test x"$OPUS_ARM_MAY_HAVE_NEON_INTR" = x"1"]) +AM_CONDITIONAL([HAVE_SSE], + [test x"$OPUS_X86_MAY_HAVE_SSE" = x"1"]) +AM_CONDITIONAL([HAVE_SSE2], + [test x"$OPUS_X86_MAY_HAVE_SSE2" = x"1"]) +AM_CONDITIONAL([HAVE_SSE4_1], + [test x"$OPUS_X86_MAY_HAVE_SSE4_1" = x"1"]) AS_IF([test x"$enable_rtcd" = x"yes"],[ AS_IF([test x"$rtcd_support" != x"no"],[ @@ -623,7 +690,7 @@ AC_MSG_NOTICE([ Fixed point debugging: ......... ${enable_fixed_point_debug} Inline Assembly Optimizations: . ${inline_optimization} External Assembly Optimizations: ${asm_optimization} - Intrinsics Optimizations.......: ${enable_intrinsics} + Intrinsics Optimizations.......: ${intrinsics_support} Run-time CPU detection: ........ ${rtcd_support} Custom modes: .................. ${enable_custom_modes} Assertion checking: ............ ${enable_assertions} diff --git a/m4/opus-intrinsics.m4 b/m4/opus-intrinsics.m4 new file mode 100644 index 0000000..c74aecd --- /dev/null +++ b/m4/opus-intrinsics.m4 @@ -0,0 +1,29 @@ +dnl opus-intrinsics.m4 +dnl macro for testing for support for compiler intrinsics, either by default or with a compiler flag + +dnl OPUS_CHECK_INTRINSICS(NAME-OF-INTRINSICS, COMPILER-FLAG-FOR-INTRINSICS, VAR-IF-PRESENT, VAR-IF-DEFAULT, TEST-PROGRAM-HEADER, TEST-PROGRAM-BODY) +AC_DEFUN([OPUS_CHECK_INTRINSICS], +[ + AC_MSG_CHECKING([if compiler supports $1 intrinsics]) + AC_LINK_IFELSE( + [AC_LANG_PROGRAM($5, $6)], + [ + $3=1 + $4=1 + AC_MSG_RESULT([yes]) + ],[ + $4=0 + AC_MSG_RESULT([no]) + AC_MSG_CHECKING([if compiler supports $1 intrinsics with $2]) + save_CFLAGS="$CFLAGS"; CFLAGS="$2 $CFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM($5, $6)], + [ + AC_MSG_RESULT([yes]) + $3=1 + ],[ + AC_MSG_RESULT([no]) + $3=0 + ]) + CFLAGS="$save_CFLAGS" + ]) +]) diff --git a/silk/x86/SigProc_FIX_sse.h b/silk/x86/SigProc_FIX_sse.h index 9a0e096..61efa8d 100644 --- a/silk/x86/SigProc_FIX_sse.h +++ b/silk/x86/SigProc_FIX_sse.h @@ -45,6 +45,12 @@ void silk_burg_modified_sse4_1( int arch /* I Run-time architecture */ ); +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define silk_burg_modified(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch) \ + ((void)(arch), silk_burg_modified_sse4_1(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch)) + +#else + extern void (*const SILK_BURG_MODIFIED_IMPL[OPUS_ARCHMASK + 1])( opus_int32 *res_nrg, /* O Residual energy */ opus_int *res_nrg_Q, /* O Residual energy Q value */ @@ -59,12 +65,22 @@ extern void (*const SILK_BURG_MODIFIED_IMPL[OPUS_ARCHMASK + 1])( # define silk_burg_modified(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch) \ ((*SILK_BURG_MODIFIED_IMPL[(arch) & OPUS_ARCHMASK])(res_nrg, res_nrg_Q, A_Q16, x, minInvGain_Q30, subfr_length, nb_subfr, D, arch)) +#endif + opus_int64 silk_inner_prod16_aligned_64_sse4_1( const opus_int16 *inVec1, const opus_int16 *inVec2, const opus_int len ); + +#if defined(OPUS_X86_PRESUME_SSE4_1) + +#define silk_inner_prod16_aligned_64(inVec1, inVec2, len, arch) \ + ((void)(arch),silk_inner_prod16_aligned_64_sse4_1(inVec1, inVec2, len)) + +#else + extern opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[OPUS_ARCHMASK + 1])( const opus_int16 *inVec1, const opus_int16 *inVec2, @@ -75,3 +91,4 @@ extern opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[OPUS_ARCHMASK + 1])( #endif #endif +#endif diff --git a/silk/x86/main_sse.h b/silk/x86/main_sse.h index f970632..afd5ec2 100644 --- a/silk/x86/main_sse.h +++ b/silk/x86/main_sse.h @@ -50,6 +50,15 @@ void silk_VQ_WMat_EC_sse4_1( opus_int L /* I number of vectors in codebook */ ); +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_VQ_WMat_EC(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L, arch) \ + ((void)(arch),silk_VQ_WMat_EC_sse4_1(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ + mu_Q9, max_gain_Q7, L)) + +#else + extern void (*const SILK_VQ_WMAT_EC_IMPL[OPUS_ARCHMASK + 1])( opus_int8 *ind, /* O index of best codebook vector */ opus_int32 *rate_dist_Q14, /* O best weighted quant error + mu * rate */ @@ -69,6 +78,8 @@ extern void (*const SILK_VQ_WMAT_EC_IMPL[OPUS_ARCHMASK + 1])( ((*SILK_VQ_WMAT_EC_IMPL[(arch) & OPUS_ARCHMASK])(ind, rate_dist_Q14, gain_Q7, in_Q14, W_Q18, cb_Q7, cb_gain_Q7, cl_Q5, \ mu_Q9, max_gain_Q7, L)) +#endif + # define OVERRIDE_silk_NSQ void silk_NSQ_sse4_1( @@ -89,6 +100,15 @@ void silk_NSQ_sse4_1( const opus_int LTP_scale_Q14 /* I LTP state scaling */ ); +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_NSQ(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_sse4_1(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#else + extern void (*const SILK_NSQ_IMPL[OPUS_ARCHMASK + 1])( const silk_encoder_state *psEncC, /* I/O Encoder State */ silk_nsq_state *NSQ, /* I/O NSQ state */ @@ -112,6 +132,8 @@ extern void (*const SILK_NSQ_IMPL[OPUS_ARCHMASK + 1])( ((*SILK_NSQ_IMPL[(arch) & OPUS_ARCHMASK])(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) +#endif + # define OVERRIDE_silk_NSQ_del_dec void silk_NSQ_del_dec_sse4_1( @@ -132,6 +154,15 @@ void silk_NSQ_del_dec_sse4_1( const opus_int LTP_scale_Q14 /* I LTP state scaling */ ); +#if defined OPUS_X86_PRESUME_SSE4_1 + +#define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14, arch) \ + ((void)(arch),silk_NSQ_del_dec_sse4_1(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ + HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) + +#else + extern void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( const silk_encoder_state *psEncC, /* I/O Encoder State */ silk_nsq_state *NSQ, /* I/O NSQ state */ @@ -155,6 +186,8 @@ extern void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( ((*SILK_NSQ_DEL_DEC_IMPL[(arch) & OPUS_ARCHMASK])(psEncC, NSQ, psIndices, x_Q3, pulses, PredCoef_Q12, LTPCoef_Q14, AR2_Q13, \ HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14)) +#endif + void silk_noise_shape_quantizer( silk_nsq_state *NSQ, /* I/O NSQ state */ opus_int signalType, /* I Signal type */ @@ -192,6 +225,11 @@ opus_int silk_VAD_GetSA_Q8_sse4_1( const opus_int16 pIn[] ); +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define silk_VAD_GetSA_Q8(psEnC, pIn, arch) ((void)(arch),silk_VAD_GetSA_Q8_sse4_1(psEnC, pIn)) + +#else + # define silk_VAD_GetSA_Q8(psEnC, pIn, arch) \ ((*SILK_VAD_GETSA_Q8_IMPL[(arch) & OPUS_ARCHMASK])(psEnC, pIn)) @@ -201,6 +239,8 @@ extern opus_int (*const SILK_VAD_GETSA_Q8_IMPL[OPUS_ARCHMASK + 1])( # define OVERRIDE_silk_warped_LPC_analysis_filter_FIX +#endif + void silk_warped_LPC_analysis_filter_FIX_sse4_1( opus_int32 state[], /* I/O State [order + 1] */ opus_int32 res_Q2[], /* O Residual signal [length] */ @@ -211,6 +251,12 @@ void silk_warped_LPC_analysis_filter_FIX_sse4_1( const opus_int order /* I Filter order (even) */ ); +#if defined(OPUS_X86_PRESUME_SSE4_1) +#define silk_warped_LPC_analysis_filter_FIX(state, res_Q2, coef_Q13, input, lambda_Q16, length, order, arch) \ + ((void)(arch),silk_warped_LPC_analysis_filter_FIX_c(state, res_Q2, coef_Q13, input, lambda_Q16, length, order)) + +#else + extern void (*const SILK_WARPED_LPC_ANALYSIS_FILTER_FIX_IMPL[OPUS_ARCHMASK + 1])( opus_int32 state[], /* I/O State [order + 1] */ opus_int32 res_Q2[], /* O Residual signal [length] */ @@ -224,5 +270,7 @@ extern void (*const SILK_WARPED_LPC_ANALYSIS_FILTER_FIX_IMPL[OPUS_ARCHMASK + 1]) # define silk_warped_LPC_analysis_filter_FIX(state, res_Q2, coef_Q13, input, lambda_Q16, length, order, arch) \ ((*SILK_WARPED_LPC_ANALYSIS_FILTER_FIX_IMPL[(arch) & OPUS_ARCHMASK])(state, res_Q2, coef_Q13, input, lambda_Q16, length, order)) +#endif + # endif #endif diff --git a/silk/x86/x86_silk_map.c b/silk/x86/x86_silk_map.c index 6747d10..ad9fef2 100644 --- a/silk/x86/x86_silk_map.c +++ b/silk/x86/x86_silk_map.c @@ -35,6 +35,10 @@ #include "pitch.h" #include "main.h" +#if !defined(OPUS_X86_PRESUME_SSE4_1) + +#if defined(FIXED_POINT) + opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[ OPUS_ARCHMASK + 1 ] )( const opus_int16 *inVec1, const opus_int16 *inVec2, @@ -42,18 +46,20 @@ opus_int64 (*const SILK_INNER_PROD16_ALIGNED_64_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_inner_prod16_aligned_64_c, /* non-sse */ silk_inner_prod16_aligned_64_c, + silk_inner_prod16_aligned_64_c, MAY_HAVE_SSE4_1( silk_inner_prod16_aligned_64 ), /* sse4.1 */ - NULL }; +#endif + opus_int (*const SILK_VAD_GETSA_Q8_IMPL[ OPUS_ARCHMASK + 1 ] )( silk_encoder_state *psEncC, const opus_int16 pIn[] ) = { silk_VAD_GetSA_Q8_c, /* non-sse */ silk_VAD_GetSA_Q8_c, + silk_VAD_GetSA_Q8_c, MAY_HAVE_SSE4_1( silk_VAD_GetSA_Q8 ), /* sse4.1 */ - NULL }; void (*const SILK_NSQ_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -75,8 +81,8 @@ void (*const SILK_NSQ_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_NSQ_c, /* non-sse */ silk_NSQ_c, + silk_NSQ_c, MAY_HAVE_SSE4_1( silk_NSQ ), /* sse4.1 */ - NULL }; void (*const SILK_VQ_WMAT_EC_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -94,8 +100,8 @@ void (*const SILK_VQ_WMAT_EC_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_VQ_WMat_EC_c, /* non-sse */ silk_VQ_WMat_EC_c, + silk_VQ_WMat_EC_c, MAY_HAVE_SSE4_1( silk_VQ_WMat_EC ), /* sse4.1 */ - NULL }; void (*const SILK_NSQ_DEL_DEC_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -117,10 +123,12 @@ void (*const SILK_NSQ_DEL_DEC_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_NSQ_del_dec_c, /* non-sse */ silk_NSQ_del_dec_c, + silk_NSQ_del_dec_c, MAY_HAVE_SSE4_1( silk_NSQ_del_dec ), /* sse4.1 */ - NULL }; +#if defined(FIXED_POINT) + void (*const SILK_WARPED_LPC_ANALYSIS_FILTER_FIX_IMPL[ OPUS_ARCHMASK + 1 ] )( opus_int32 state[], /* I/O State [order + 1] */ opus_int32 res_Q2[], /* O Residual signal [length] */ @@ -132,8 +140,8 @@ void (*const SILK_WARPED_LPC_ANALYSIS_FILTER_FIX_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_warped_LPC_analysis_filter_FIX_c, /* non-sse */ silk_warped_LPC_analysis_filter_FIX_c, + silk_warped_LPC_analysis_filter_FIX_c, MAY_HAVE_SSE4_1( silk_warped_LPC_analysis_filter_FIX ), /* sse4.1 */ - NULL }; void (*const SILK_BURG_MODIFIED_IMPL[ OPUS_ARCHMASK + 1 ] )( @@ -149,6 +157,9 @@ void (*const SILK_BURG_MODIFIED_IMPL[ OPUS_ARCHMASK + 1 ] )( ) = { silk_burg_modified_c, /* non-sse */ silk_burg_modified_c, + silk_burg_modified_c, MAY_HAVE_SSE4_1( silk_burg_modified ), /* sse4.1 */ - NULL }; + +#endif +#endif diff --git a/win32/VS2010/celt.vcxproj b/win32/VS2010/celt.vcxproj index f107fec..e068fbe 100644 --- a/win32/VS2010/celt.vcxproj +++ b/win32/VS2010/celt.vcxproj @@ -37,6 +37,12 @@ + + + + + + @@ -67,6 +73,9 @@ + + + @@ -141,7 +150,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - ..\;..\..\include;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\celt;$(ProjectDir)\..\..\silk;$(ProjectDir)\..\..\silk\float;$(ProjectDir)\..\..\silk\fixed;%(AdditionalIncludeDirectories) MultiThreadedDebug @@ -168,7 +177,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;WIN64;_DEBUG;_LIB;%(PreprocessorDefinitions) - ..\;..\..\include;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\celt;$(ProjectDir)\..\..\silk;$(ProjectDir)\..\..\silk\float;$(ProjectDir)\..\..\silk\fixed;%(AdditionalIncludeDirectories) MultiThreadedDebug @@ -196,7 +205,7 @@ true true HAVE_CONFIG_H;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - ..\;..\..\include;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\celt;$(ProjectDir)\..\..\silk;$(ProjectDir)\..\..\silk\float;$(ProjectDir)\..\..\silk\fixed;%(AdditionalIncludeDirectories) MultiThreaded @@ -227,7 +236,7 @@ true true HAVE_CONFIG_H;WIN32;WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) - ..\;..\..\include;%(AdditionalIncludeDirectories) + $(ProjectDir)\..\;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\celt;$(ProjectDir)\..\..\silk;$(ProjectDir)\..\..\silk\float;$(ProjectDir)\..\..\silk\fixed;%(AdditionalIncludeDirectories) MultiThreaded diff --git a/win32/VS2010/celt.vcxproj.filters b/win32/VS2010/celt.vcxproj.filters index e3a1d97..e9948fa 100644 --- a/win32/VS2010/celt.vcxproj.filters +++ b/win32/VS2010/celt.vcxproj.filters @@ -69,6 +69,24 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + @@ -158,5 +176,14 @@ Header Files + + Header Files + + + Header Files + + + Header Files + \ No newline at end of file diff --git a/win32/VS2010/silk_common.vcxproj b/win32/VS2010/silk_common.vcxproj index 9cf5f48..d3d077d 100644 --- a/win32/VS2010/silk_common.vcxproj +++ b/win32/VS2010/silk_common.vcxproj @@ -88,7 +88,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk/float;../../win32;../../celt;../../include + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk/float;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include MultiThreadedDebug @@ -118,7 +118,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;WIN64;_DEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk/float;../../win32;../../celt;../../include + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk/float;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include MultiThreadedDebug @@ -149,7 +149,7 @@ true true HAVE_CONFIG_H;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk/float;../../win32;../../celt;../../include + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk/float;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include MultiThreaded Fast @@ -184,7 +184,7 @@ true true HAVE_CONFIG_H;WIN32;WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk/float;../../win32;../../celt;../../include + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk/float;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include MultiThreaded Fast @@ -212,6 +212,8 @@ + + @@ -311,8 +313,13 @@ + + + + + - + \ No newline at end of file diff --git a/win32/VS2010/silk_common.vcxproj.filters b/win32/VS2010/silk_common.vcxproj.filters index 30db48e..341180b 100644 --- a/win32/VS2010/silk_common.vcxproj.filters +++ b/win32/VS2010/silk_common.vcxproj.filters @@ -81,6 +81,12 @@ Header Files + + Header Files + + + Header Files + @@ -311,5 +317,20 @@ Source Files + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + - + \ No newline at end of file diff --git a/win32/VS2010/silk_fixed.vcxproj b/win32/VS2010/silk_fixed.vcxproj index 5ea1a91..522101e 100644 --- a/win32/VS2010/silk_fixed.vcxproj +++ b/win32/VS2010/silk_fixed.vcxproj @@ -86,7 +86,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk;../../win32;../../celt;../../include;../win32 + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include;$(ProjectDir)/../win32 MultiThreadedDebug @@ -104,7 +104,7 @@ Level3 Disabled HAVE_CONFIG_H;WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk;../../win32;../../celt;../../include;../win32 + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include;$(ProjectDir)/../win32 MultiThreadedDebug @@ -123,7 +123,7 @@ true true HAVE_CONFIG_H;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk;../../win32;../../celt;../../include;../win32 + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include;$(ProjectDir)/../win32 MultiThreaded @@ -145,7 +145,7 @@ true true HAVE_CONFIG_H;WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) - ../../silk/fixed;../../silk;../../win32;../../celt;../../include;../win32 + $(ProjectDir)/../..;$(ProjectDir)/../../silk/fixed;$(ProjectDir)/../../silk;$(ProjectDir)/../../win32;$(ProjectDir)/../../celt;$(ProjectDir)/../../include;$(ProjectDir)/../win32 MultiThreaded @@ -191,8 +191,11 @@ + + + - + \ No newline at end of file diff --git a/win32/VS2010/silk_fixed.vcxproj.filters b/win32/VS2010/silk_fixed.vcxproj.filters index 6897930..c2327eb 100644 --- a/win32/VS2010/silk_fixed.vcxproj.filters +++ b/win32/VS2010/silk_fixed.vcxproj.filters @@ -18,16 +18,16 @@ Header Files - + Header Files - + Header Files - + Header Files - + Header Files @@ -107,5 +107,14 @@ Source Files + + Source Files + + + Source Files + + + Source Files + \ No newline at end of file diff --git a/win32/config.h b/win32/config.h index 46ff699..10fbf33 100644 --- a/win32/config.h +++ b/win32/config.h @@ -35,9 +35,28 @@ POSSIBILITY OF SUCH DAMAGE. #define OPUS_BUILD 1 -/* Enable SSE functions, if compiled with SSE/SSE2 (note that AMD64 implies SSE2) */ -#if defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP >= 1)) -#define __SSE__ 1 +#if defined(_M_IX86) || defined(_M_X64) +/* Can always build with SSE intrinsics (no special compiler flags necessary) */ +#define OPUS_X86_MAY_HAVE_SSE +#define OPUS_X86_MAY_HAVE_SSE2 +#define OPUS_X86_MAY_HAVE_SSE4_1 + +/* Presume SSE functions, if compiled with SSE/SSE2/AVX (note that AMD64 implies SSE2, and AVX + implies SSE4.1) */ +#if defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP >= 1)) || defined(__AVX__) +#define OPUS_X86_PRESUME_SSE 1 +#endif +#if defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || defined(__AVX__) +#define OPUS_X86_PRESUME_SSE2 1 +#endif +#if defined(__AVX__) +#define OPUS_X86_PRESUME_SSE4_1 1 +#endif + +#if !defined(OPUS_X86_PRESUME_SSE4_1) || !defined(OPUS_X86_PRESUME_SSE2) || !defined(OPUS_X86_PRESUME_SSE) +#define OPUS_HAVE_RTCD 1 +#endif + #endif #include "version.h"