[opus] [PATCH 2/5] Optimize fixed-point celt_fir_c() for ARM NEON
Linfeng Zhang
linfengz at google.com
Thu Jul 14 00:48:59 UTC 2016
Create the fixed-point intrinsics optimization celt_fir_neon() for ARM NEON.
Create test tests/test_unit_optimization to unit test the optimization.
---
.gitignore | 1 +
Makefile.am | 39 ++++-
celt/arm/arm_celt_map.c | 17 +++
celt/arm/celt_lpc_arm.h | 65 ++++++++
celt/arm/celt_lpc_neon_intr.c | 254 ++++++++++++++++++++++++++++++++
celt/celt_lpc.h | 5 +
celt/tests/test_unit_dft.c | 1 +
celt/tests/test_unit_mathops.c | 1 +
celt/tests/test_unit_mdct.c | 1 +
celt/tests/test_unit_optimization_lpc.c | 96 ++++++++++++
celt/tests/test_unit_rotation.c | 1 +
celt_headers.mk | 1 +
celt_sources.mk | 1 +
tests/test_unit_optimization.c | 62 ++++++++
14 files changed, 541 insertions(+), 4 deletions(-)
create mode 100644 celt/arm/celt_lpc_arm.h
create mode 100644 celt/arm/celt_lpc_neon_intr.c
create mode 100644 celt/tests/test_unit_optimization_lpc.c
create mode 100644 tests/test_unit_optimization.c
diff --git a/.gitignore b/.gitignore
index 33127c9..05d0582 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,7 @@ tests/test_opus_api
tests/test_opus_decode
tests/test_opus_encode
tests/test_opus_padding
+tests/test_unit_optimization
celt/arm/armopts.s
celt/dump_modes/dump_modes
celt/tests/test_unit_cwrs32
diff --git a/Makefile.am b/Makefile.am
index 7a69114..2bfb923 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -84,9 +84,36 @@ pkginclude_HEADERS = include/opus.h include/opus_multistream.h include/opus_type
noinst_HEADERS = $(OPUS_HEAD) $(SILK_HEAD) $(CELT_HEAD)
if EXTRA_PROGRAMS
-noinst_PROGRAMS = opus_demo repacketizer_demo opus_compare tests/test_opus_api tests/test_opus_encode tests/test_opus_decode tests/test_opus_padding celt/tests/test_unit_cwrs32 celt/tests/test_unit_dft celt/tests/test_unit_entropy celt/tests/test_unit_laplace celt/tests/test_unit_mathops celt/tests/test_unit_mdct celt/tests/test_unit_rotation celt/tests/test_unit_types
-
-TESTS = celt/tests/test_unit_types celt/tests/test_unit_mathops celt/tests/test_unit_entropy celt/tests/test_unit_laplace celt/tests/test_unit_dft celt/tests/test_unit_mdct celt/tests/test_unit_rotation celt/tests/test_unit_cwrs32 tests/test_opus_api tests/test_opus_decode tests/test_opus_encode tests/test_opus_padding
+noinst_PROGRAMS = opus_demo \
+ repacketizer_demo \
+ opus_compare \
+ celt/tests/test_unit_cwrs32 \
+ celt/tests/test_unit_dft \
+ celt/tests/test_unit_entropy \
+ celt/tests/test_unit_laplace \
+ celt/tests/test_unit_mathops \
+ celt/tests/test_unit_mdct \
+ celt/tests/test_unit_rotation \
+ celt/tests/test_unit_types \
+ tests/test_opus_api \
+ tests/test_opus_encode \
+ tests/test_opus_decode \
+ tests/test_opus_padding \
+ tests/test_unit_optimization
+
+TESTS = celt/tests/test_unit_types \
+ celt/tests/test_unit_mathops \
+ celt/tests/test_unit_entropy \
+ celt/tests/test_unit_laplace \
+ celt/tests/test_unit_dft \
+ celt/tests/test_unit_mdct \
+ celt/tests/test_unit_rotation \
+ celt/tests/test_unit_cwrs32 \
+ tests/test_opus_api \
+ tests/test_opus_decode \
+ tests/test_opus_encode \
+ tests/test_opus_padding \
+ tests/test_unit_optimization
opus_demo_SOURCES = src/opus_demo.c
@@ -111,6 +138,9 @@ tests_test_opus_decode_LDADD = libopus.la $(NE10_LIBS) $(LIBM)
tests_test_opus_padding_SOURCES = tests/test_opus_padding.c tests/test_opus_common.h
tests_test_opus_padding_LDADD = libopus.la $(NE10_LIBS) $(LIBM)
+tests_test_unit_optimization_SOURCES = tests/test_unit_optimization.c
+tests_test_unit_optimization_LDADD = libopus.la $(NE10_LIBS) $(LIBM)
+
celt_tests_test_unit_cwrs32_SOURCES = celt/tests/test_unit_cwrs32.c
celt_tests_test_unit_cwrs32_LDADD = $(LIBM)
@@ -276,7 +306,8 @@ $(CELT_SOURCES_ARM_ASM:%.s=%-gnu.S): $(top_srcdir)/celt/arm/arm2gnu.pl
OPT_UNIT_TEST_OBJ = $(celt_tests_test_unit_mathops_SOURCES:.c=.o) \
$(celt_tests_test_unit_rotation_SOURCES:.c=.o) \
$(celt_tests_test_unit_mdct_SOURCES:.c=.o) \
- $(celt_tests_test_unit_dft_SOURCES:.c=.o)
+ $(celt_tests_test_unit_dft_SOURCES:.c=.o) \
+ $(tests_test_unit_optimization_SOURCES:.c=.o)
if HAVE_SSE
SSE_OBJ = $(CELT_SOURCES_SSE:.c=.lo)
diff --git a/celt/arm/arm_celt_map.c b/celt/arm/arm_celt_map.c
index 4d4d069..74869ab 100644
--- a/celt/arm/arm_celt_map.c
+++ b/celt/arm/arm_celt_map.c
@@ -29,6 +29,7 @@
#include "config.h"
#endif
+#include "celt_lpc.h"
#include "pitch.h"
#include "kiss_fft.h"
#include "mdct.h"
@@ -39,6 +40,22 @@
# if ((defined(OPUS_ARM_MAY_HAVE_NEON) && !defined(OPUS_ARM_PRESUME_NEON)) || \
(defined(OPUS_ARM_MAY_HAVE_MEDIA) && !defined(OPUS_ARM_PRESUME_MEDIA)) || \
(defined(OPUS_ARM_MAY_HAVE_EDSP) && !defined(OPUS_ARM_PRESUME_EDSP)))
+void celt_fir_neon(
+ const opus_val16 *_x,
+ const opus_val16 *num,
+ opus_val16 *_y,
+ int N,
+ int ord,
+ int arch);
+
+void (*const CELT_FIR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *,
+ const opus_val16 *, opus_val16 *, int, int, int) = {
+ celt_fir_c, /* ARMv4 */
+ celt_fir_c, /* EDSP */
+ celt_fir_c, /* Media */
+ MAY_HAVE_NEON(celt_fir) /* NEON */
+};
+
opus_val32 (*const CELT_PITCH_XCORR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *,
const opus_val16 *, opus_val32 *, int , int) = {
celt_pitch_xcorr_c, /* ARMv4 */
diff --git a/celt/arm/celt_lpc_arm.h b/celt/arm/celt_lpc_arm.h
new file mode 100644
index 0000000..101df3d
--- /dev/null
+++ b/celt/arm/celt_lpc_arm.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016 Google Inc. */
+/*
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#if !defined(CELT_LPC_ARM_H)
+# define CELT_LPC_ARM_H
+
+# include "armcpu.h"
+
+# if defined(FIXED_POINT)
+
+# if defined(OPUS_ARM_MAY_HAVE_NEON)
+void celt_fir_neon(
+ const opus_val16 *_x,
+ const opus_val16 *num,
+ opus_val16 *_y,
+ int N,
+ int ord,
+ int arch);
+# endif
+
+# if !defined(OPUS_HAVE_RTCD)
+# define OVERRIDE_CELT_FIR (1)
+# define celt_fir(x, num, y, N, ord, arch) \
+ ((void)(arch),PRESUME_NEON(celt_fir)(x, num, y, N, ord, arch))
+# endif
+
+#if !defined(OVERRIDE_CELT_FIR)
+/*Is run-time CPU detection enabled on this platform?*/
+# if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_ASM) \
+ || (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) \
+ && !defined(OPUS_ARM_PRESUME_NEON_INTR)))
+extern void (*const CELT_FIR_IMPL[OPUS_ARCHMASK+1])(const opus_val16 *,
+ const opus_val16 *, opus_val16 *, int, int, int);
+
+# define OVERRIDE_CELT_FIR
+# define celt_fir(x, num, y, N, ord, arch) \
+ ((*CELT_FIR_IMPL[(arch)&OPUS_ARCHMASK])(x, num, y, N, ord, arch))
+# endif
+#endif
+#endif /* end FIXED_POINT */
+
+#endif /* end CELT_LPC_ARM_H */
diff --git a/celt/arm/celt_lpc_neon_intr.c b/celt/arm/celt_lpc_neon_intr.c
new file mode 100644
index 0000000..4715d0b
--- /dev/null
+++ b/celt/arm/celt_lpc_neon_intr.c
@@ -0,0 +1,254 @@
+/* Copyright (c) 2016 Google Inc. */
+/**
+ @file celt_lpc_neon_intr.c
+ @brief ARM Neon Intrinsic optimizations for celt lpc functions
+ */
+
+/*
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <arm_neon.h>
+#include "celt_lpc.h"
+#include "stack_alloc.h"
+
+#if defined(FIXED_POINT)
+
+void celt_fir_neon(
+ const opus_val16 *_x,
+ const opus_val16 *num,
+ opus_val16 *y,
+ int N,
+ int ord,
+ int arch)
+{
+ int i,j;
+ const int leftover = N & 7;
+ const opus_val16 *x = _x-ord;
+ VARDECL(opus_val16, rnum);
+ SAVE_STACK;
+ /* Extend rnum by 3 zeros to handle the case that (ord % 4) is non-zero. */
+ ALLOC(rnum, ord+3, opus_val16);
+ for (i=0;i<ord-3;i+=4)
+ vst1_s16(rnum+i, vrev64_s16(vld1_s16(num+ord-i-4)));
+ for (;i<ord;i++)
+ rnum[i] = num[ord-i-1];
+ rnum[ord] = rnum[ord+1] = rnum[ord+2] = 0;
+ (void)arch;
+
+#ifdef SMALL_FOOTPRINT
+ for (i=0;i<N-7;i+=8)
+ {
+ int16x8_t x_s16x8 = vld1q_s16(_x+i);
+ int32x4_t sum0_s32x4 = vshll_n_s16(vget_low_s16 (x_s16x8), SIG_SHIFT);
+ int32x4_t sum1_s32x4 = vshll_n_s16(vget_high_s16(x_s16x8), SIG_SHIFT);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ x_s16x8 = vld1q_s16(x+i+j+0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 0);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 0);
+ x_s16x8 = vld1q_s16(x+i+j+1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 1);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 1);
+ x_s16x8 = vld1q_s16(x+i+j+2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 2);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 2);
+ x_s16x8 = vld1q_s16(x+i+j+3);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 3);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 3);
+ }
+ vst1q_s16(y+i, vcombine_s16(vqrshrn_n_s32(sum0_s32x4, SIG_SHIFT), vqrshrn_n_s32(sum1_s32x4, SIG_SHIFT)));
+ }
+ if (leftover)
+ {
+ if (leftover > 4)
+ {
+ int16x8_t x_s16x8 = vld1q_s16(_x+i);
+ int32x4_t sum0_s32x4 = vshll_n_s16(vget_low_s16 (x_s16x8), SIG_SHIFT);
+ int32x4_t sum1_s32x4 = vshll_n_s16(vget_high_s16(x_s16x8), SIG_SHIFT);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ x_s16x8 = vld1q_s16(x+i+j+0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 0);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 0);
+ x_s16x8 = vld1q_s16(x+i+j+1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 1);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 1);
+ x_s16x8 = vld1q_s16(x+i+j+2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 2);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 2);
+ x_s16x8 = vld1q_s16(x+i+j+3);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 3);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 3);
+ }
+ const int16x8_t y_s16x8 = vcombine_s16(vqrshrn_n_s32(sum0_s32x4, SIG_SHIFT), vqrshrn_n_s32(sum1_s32x4, SIG_SHIFT));
+ vst1_s16(y+i, vget_low_s16(y_s16x8));
+ vst1q_lane_s16(y+i+4, y_s16x8, 4);
+ if (leftover >= 6)
+ {
+ vst1q_lane_s16(y+i+5, y_s16x8, 5);
+ if (leftover == 7)
+ {
+ vst1q_lane_s16(y+i+6, y_s16x8, 6);
+ }
+ }
+ }
+ else {
+ int32x4_t sum0_s32x4 = vshll_n_s16(vld1_s16(_x+i), SIG_SHIFT);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+0), rnum_s16x4, 0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+1), rnum_s16x4, 1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+2), rnum_s16x4, 2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+3), rnum_s16x4, 3);
+ }
+ const int16x4_t y_s16x4 = vqrshrn_n_s32(sum0_s32x4, SIG_SHIFT);
+ if (leftover == 4)
+ {
+ vst1_s16(y+i, y_s16x4);
+ }
+ else
+ {
+ vst1_lane_s16(y+i, y_s16x4, 0);
+ if (leftover >= 2)
+ {
+ vst1_lane_s16(y+i+1, y_s16x4, 1);
+ if (leftover == 3)
+ {
+ vst1_lane_s16(y+i+2, y_s16x4, 2);
+ }
+ }
+ }
+ }
+ }
+#else
+ for (i=0;i<N-7;i+=8)
+ {
+ int32x4_t sum0_s32x4, sum1_s32x4;
+ sum0_s32x4 = sum1_s32x4 = vdupq_n_s32(0);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ int16x8_t x_s16x8 = vld1q_s16(x+i+j+0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 0);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 0);
+ x_s16x8 = vld1q_s16(x+i+j+1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 1);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 1);
+ x_s16x8 = vld1q_s16(x+i+j+2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 2);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 2);
+ x_s16x8 = vld1q_s16(x+i+j+3);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 3);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 3);
+ }
+ sum0_s32x4 = vrshrq_n_s32(sum0_s32x4, SIG_SHIFT);
+ sum1_s32x4 = vrshrq_n_s32(sum1_s32x4, SIG_SHIFT);
+ const int16x8_t x_s16x8 = vld1q_s16(_x+i);
+ sum0_s32x4 = vaddw_s16(sum0_s32x4, vget_low_s16 (x_s16x8));
+ sum1_s32x4 = vaddw_s16(sum1_s32x4, vget_high_s16(x_s16x8));
+ vst1q_s16(y+i, vcombine_s16(vqmovn_s32(sum0_s32x4), vqmovn_s32(sum1_s32x4)));
+ }
+ if (leftover)
+ {
+ if (leftover > 4)
+ {
+ int32x4_t sum0_s32x4, sum1_s32x4;
+ sum0_s32x4 = sum1_s32x4 = vdupq_n_s32(0);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ int16x8_t x_s16x8 = vld1q_s16(x+i+j+0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 0);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 0);
+ x_s16x8 = vld1q_s16(x+i+j+1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 1);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 1);
+ x_s16x8 = vld1q_s16(x+i+j+2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 2);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 2);
+ x_s16x8 = vld1q_s16(x+i+j+3);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vget_low_s16 (x_s16x8), rnum_s16x4, 3);
+ sum1_s32x4 = vmlal_lane_s16(sum1_s32x4, vget_high_s16(x_s16x8), rnum_s16x4, 3);
+ }
+ sum0_s32x4 = vrshrq_n_s32(sum0_s32x4, SIG_SHIFT);
+ sum1_s32x4 = vrshrq_n_s32(sum1_s32x4, SIG_SHIFT);
+ const int16x8_t x_s16x8 = vld1q_s16(_x+i);
+ sum0_s32x4 = vaddw_s16(sum0_s32x4, vget_low_s16 (x_s16x8));
+ sum1_s32x4 = vaddw_s16(sum1_s32x4, vget_high_s16(x_s16x8));
+ const int16x8_t y_s16x8 = vcombine_s16(vqmovn_s32(sum0_s32x4), vqmovn_s32(sum1_s32x4));
+ vst1_s16(y+i, vget_low_s16(y_s16x8));
+ vst1q_lane_s16(y+i+4, y_s16x8, 4);
+ if (leftover >= 6)
+ {
+ vst1q_lane_s16(y+i+5, y_s16x8, 5);
+ if (leftover == 7)
+ {
+ vst1q_lane_s16(y+i+6, y_s16x8, 6);
+ }
+ }
+ }
+ else {
+ int32x4_t sum0_s32x4 = vdupq_n_s32(0);
+ for (j=0;j<ord;j+=4)
+ {
+ const int16x4_t rnum_s16x4 = vld1_s16(rnum+j);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+0), rnum_s16x4, 0);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+1), rnum_s16x4, 1);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+2), rnum_s16x4, 2);
+ sum0_s32x4 = vmlal_lane_s16(sum0_s32x4, vld1_s16(x+i+j+3), rnum_s16x4, 3);
+ }
+ sum0_s32x4 = vrshrq_n_s32(sum0_s32x4, SIG_SHIFT);
+ sum0_s32x4 = vaddw_s16(sum0_s32x4, vld1_s16(_x+i));
+ const int16x4_t y_s16x4 = vqmovn_s32(sum0_s32x4);
+ if (leftover == 4)
+ {
+ vst1_s16(y+i, y_s16x4);
+ }
+ else
+ {
+ vst1_lane_s16(y+i, y_s16x4, 0);
+ if (leftover >= 2)
+ {
+ vst1_lane_s16(y+i+1, y_s16x4, 1);
+ if (leftover == 3)
+ {
+ vst1_lane_s16(y+i+2, y_s16x4, 2);
+ }
+ }
+ }
+ }
+ }
+#endif
+ RESTORE_STACK;
+}
+
+#endif
diff --git a/celt/celt_lpc.h b/celt/celt_lpc.h
index a4c5fd6..76a73c0 100644
--- a/celt/celt_lpc.h
+++ b/celt/celt_lpc.h
@@ -35,6 +35,11 @@
#include "x86/celt_lpc_sse.h"
#endif
+#if ((defined(OPUS_ARM_ASM) && defined(FIXED_POINT)) \
+ || defined(OPUS_ARM_MAY_HAVE_NEON_INTR))
+#include "arm/celt_lpc_arm.h"
+#endif
+
#define LPC_ORDER 24
void _celt_lpc(opus_val16 *_lpc, const opus_val32 *ac, int p);
diff --git a/celt/tests/test_unit_dft.c b/celt/tests/test_unit_dft.c
index 6166eb0..582618e 100644
--- a/celt/tests/test_unit_dft.c
+++ b/celt/tests/test_unit_dft.c
@@ -52,6 +52,7 @@
# include "celt_lpc.c"
# include "pitch.c"
# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
+# include "arm/celt_lpc_neon_intr.c"
# include "arm/celt_neon_intr.c"
# if defined(HAVE_ARM_NE10)
# include "mdct.c"
diff --git a/celt/tests/test_unit_mathops.c b/celt/tests/test_unit_mathops.c
index fd3319d..da92f16 100644
--- a/celt/tests/test_unit_mathops.c
+++ b/celt/tests/test_unit_mathops.c
@@ -66,6 +66,7 @@
#elif defined(OPUS_ARM_ASM) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
# include "arm/armcpu.c"
# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
+# include "arm/celt_lpc_neon_intr.c"
# include "arm/celt_neon_intr.c"
# if defined(HAVE_ARM_NE10)
# include "kiss_fft.c"
diff --git a/celt/tests/test_unit_mdct.c b/celt/tests/test_unit_mdct.c
index 8dbb9ca..0658c7a 100644
--- a/celt/tests/test_unit_mdct.c
+++ b/celt/tests/test_unit_mdct.c
@@ -53,6 +53,7 @@
# include "pitch.c"
# include "celt_lpc.c"
# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
+# include "arm/celt_lpc_neon_intr.c"
# include "arm/celt_neon_intr.c"
# if defined(HAVE_ARM_NE10)
# include "arm/celt_ne10_fft.c"
diff --git a/celt/tests/test_unit_optimization_lpc.c b/celt/tests/test_unit_optimization_lpc.c
new file mode 100644
index 0000000..7247046
--- /dev/null
+++ b/celt/tests/test_unit_optimization_lpc.c
@@ -0,0 +1,96 @@
+/* Copyright (c) 2016 Google Inc. */
+/*
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#define SKIP_CONFIG_H
+
+#ifndef CUSTOM_MODES
+# define CUSTOM_MODES
+#endif
+
+#include <stdio.h>
+#include <string.h>
+
+#ifndef CELT_C
+# define CELT_C
+#endif
+#include "celt_lpc.h"
+#include "modes.h"
+
+#ifdef FIXED_POINT
+
+#define MAX_ORDER 32
+
+static int test_fir(int arch)
+{
+ opus_val16 x[MAX_PERIOD+MAX_ORDER];
+ opus_val16 num[MAX_ORDER];
+ opus_val16 yorg[MAX_PERIOD], yopt[MAX_PERIOD];
+ int N, ord;
+ unsigned int i;
+
+ printf("%50s", "celt_fir() ...");
+ for(ord=0;ord<=MAX_ORDER;ord++)
+ {
+ for(N=ord;N<=MAX_PERIOD;N++) /* N is larger than or equal to ord. */
+ {
+ for (i=0;i<MAX_PERIOD+MAX_ORDER;++i)
+ {
+ x[i] = (rand() % 32767) - 16384;
+ }
+ for (i=0;i<MAX_PERIOD;++i)
+ {
+ yorg[i] = (rand() % 32767) - 16384;
+ }
+ for (i=0;i<MAX_ORDER;++i)
+ {
+ num[i] = (rand() % 32767) - 16384;
+ }
+ memcpy(yopt, yorg, sizeof(yorg));
+
+ celt_fir_c(x+MAX_ORDER, num, yorg, N, ord, arch);
+ celt_fir (x+MAX_ORDER, num, yopt, N, ord, arch);
+ if (memcmp(yorg, yopt, sizeof(yorg)))
+ {
+ printf("ord=%2d N=%3d failed!\nError in lpc unit test!!!\n", ord, N);
+ for (i=0;i<sizeof(yorg) / sizeof(*yorg);i++)
+ {
+ if (yorg[i] != yopt[i])
+ {
+ printf("yorg[%3d]=%d, yopt[%3d]=%d\n", i, yorg[i], i, yopt[i]);
+ }
+ }
+ return -1;
+ }
+ }
+ }
+ printf(" passed!\n");
+ return 0;
+}
+#endif /* FIXED_POINT */
diff --git a/celt/tests/test_unit_rotation.c b/celt/tests/test_unit_rotation.c
index 1080c20..3a85a29 100644
--- a/celt/tests/test_unit_rotation.c
+++ b/celt/tests/test_unit_rotation.c
@@ -64,6 +64,7 @@
#elif defined(OPUS_ARM_ASM) || defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
# include "arm/armcpu.c"
# if defined(OPUS_ARM_MAY_HAVE_NEON_INTR)
+# include "arm/celt_lpc_neon_intr.c"
# include "arm/celt_neon_intr.c"
# if defined(HAVE_ARM_NE10)
# include "kiss_fft.c"
diff --git a/celt_headers.mk b/celt_headers.mk
index c9df94b..36ae290 100644
--- a/celt_headers.mk
+++ b/celt_headers.mk
@@ -34,6 +34,7 @@ celt/static_modes_fixed.h \
celt/static_modes_float_arm_ne10.h \
celt/static_modes_fixed_arm_ne10.h \
celt/arm/armcpu.h \
+celt/arm/celt_lpc_arm.h \
celt/arm/fixed_armv4.h \
celt/arm/fixed_armv5e.h \
celt/arm/fixed_arm64.h \
diff --git a/celt_sources.mk b/celt_sources.mk
index 2ffe99a..37c0129 100644
--- a/celt_sources.mk
+++ b/celt_sources.mk
@@ -37,6 +37,7 @@ CELT_AM_SOURCES_ARM_ASM = \
celt/arm/armopts.s.in
CELT_SOURCES_ARM_NEON_INTR = \
+celt/arm/celt_lpc_neon_intr.c \
celt/arm/celt_neon_intr.c
CELT_SOURCES_ARM_NE10= \
diff --git a/tests/test_unit_optimization.c b/tests/test_unit_optimization.c
new file mode 100644
index 0000000..7eeab38
--- /dev/null
+++ b/tests/test_unit_optimization.c
@@ -0,0 +1,62 @@
+/* Copyright (c) 2016 Google Inc. */
+/*
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ - Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include "stack_alloc.h"
+
+#define SKIP_CONFIG_H
+
+#ifndef CUSTOM_MODES
+#define CUSTOM_MODES
+#endif
+
+#ifdef FIXED_POINT
+
+# include "celt/tests/test_unit_optimization_lpc.c"
+
+#endif
+
+int main(void)
+{
+ int result = 0; /* 0: passed; other: failed */
+ ALLOC_STACK;
+#ifdef FIXED_POINT
+ int arch = opus_select_arch();
+#endif /* FIXED_POINT */
+ int count = 10;
+
+ while (!result && count--) {
+ printf("\n--------------------------- Testing optimization ---------------------------\n");
+#ifdef FIXED_POINT
+ result |= test_fir(arch);
+#endif /* FIXED_POINT */
+ }
+ return result;
+}
--
2.8.0.rc3.226.g39d4020
More information about the opus
mailing list