[opus] Patches for adding 120 ms encoding
Felicia Lim
flim at google.com
Wed Jun 1 16:58:19 UTC 2016
Hi all,
I've just realized that there's a better and simpler way of doing this
which ensures that analysis and selection of the mode/bandwidth etc is done
on the whole 120 ms frame. I believe that the logic and threshold values
for making these decisions are still valid for 120 ms, but I might be
missing something? Please find my updated patches attached.
Thanks,
Felicia
On Tue, May 31, 2016 at 7:05 PM Felicia Lim <flim at google.com> wrote:
> Hi all,
>
> We (WebRTC/Google) would like to extend Opus to natively support 120 ms
> encoding instead of relying on repacketization as a post processing step.
> This is to ensure that a valid 120 ms packet is always available. I've
> attached a couple of patches to add this to opus_encoder(), based on the
> internal repacketization process carried out by 60 ms CELT. We intend to
> extend this later for the multistream encoder as well. The first patch
> refactors out the internal subframe encoding and repacketizing, and the
> second patch actually adds the 120 ms support.
>
> Any thoughts would be appreciated.
>
> Thanks,
> Felicia
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.xiph.org/pipermail/opus/attachments/20160601/0a401c07/attachment.html>
-------------- next part --------------
From 107067d49c27eb760da5c000999dedcb992be905 Mon Sep 17 00:00:00 2001
From: Felicia Lim <flim at google.com>
Date: Tue, 31 May 2016 14:46:17 +0200
Subject: [PATCH 2/2] Extend support for 120 ms
---
include/opus_defines.h | 3 +++
src/opus_demo.c | 12 ++++++++----
src/opus_encoder.c | 44 +++++++++++++++++++++++++++++++-------------
tests/test_opus_api.c | 3 +++
4 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/include/opus_defines.h b/include/opus_defines.h
index 315412d..466c6e2 100644
--- a/include/opus_defines.h
+++ b/include/opus_defines.h
@@ -208,6 +208,7 @@ extern "C" {
#define OPUS_FRAMESIZE_20_MS 5004 /**< Use 20 ms frames */
#define OPUS_FRAMESIZE_40_MS 5005 /**< Use 40 ms frames */
#define OPUS_FRAMESIZE_60_MS 5006 /**< Use 60 ms frames */
+#define OPUS_FRAMESIZE_120_MS 5007 /**< Use 120 ms frames */
/**@}*/
@@ -566,6 +567,7 @@ extern "C" {
* <dt>OPUS_FRAMESIZE_20_MS</dt><dd>Use 20 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_40_MS</dt><dd>Use 40 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_60_MS</dt><dd>Use 60 ms frames.</dd>
+ * <dt>OPUS_FRAMESIZE_120_MS</dt><dd>Use 120 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_VARIABLE</dt><dd>Optimize the frame size dynamically.</dd>
* </dl>
* @hideinitializer */
@@ -581,6 +583,7 @@ extern "C" {
* <dt>OPUS_FRAMESIZE_20_MS</dt><dd>Use 20 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_40_MS</dt><dd>Use 40 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_60_MS</dt><dd>Use 60 ms frames.</dd>
+ * <dt>OPUS_FRAMESIZE_120_MS</dt><dd>Use 120 ms frames.</dd>
* <dt>OPUS_FRAMESIZE_VARIABLE</dt><dd>Optimize the frame size dynamically.</dd>
* </dl>
* @hideinitializer */
diff --git a/src/opus_demo.c b/src/opus_demo.c
index 9e99a3b..b79d2f8 100644
--- a/src/opus_demo.c
+++ b/src/opus_demo.c
@@ -40,7 +40,7 @@
#include "opus_private.h"
#include "opus_multistream.h"
-#define MAX_PACKET 1500
+#define MAX_PACKET 7660
void print_usage( char* argv[] )
{
@@ -56,7 +56,7 @@ void print_usage( char* argv[] )
fprintf(stderr, "-cvbr : enable constrained variable bitrate; default: unconstrained\n" );
fprintf(stderr, "-variable-duration : enable frames of variable duration (experts only); default: disabled\n" );
fprintf(stderr, "-bandwidth <NB|MB|WB|SWB|FB> : audio bandwidth (from narrowband to fullband); default: sampling rate\n" );
- fprintf(stderr, "-framesize <2.5|5|10|20|40|60> : frame size in ms; default: 20 \n" );
+ fprintf(stderr, "-framesize <2.5|5|10|20|40|60|120> : frame size in ms; default: 20 \n" );
fprintf(stderr, "-max_payload <bytes> : maximum payload size in bytes, default: 1024\n" );
fprintf(stderr, "-complexity <comp> : complexity, 0 (lowest) ... 10 (highest); default: 10\n" );
fprintf(stderr, "-inbandfec : enable SILK inband FEC\n" );
@@ -382,9 +382,11 @@ int main(int argc, char *argv[])
frame_size = sampling_rate/25;
else if (strcmp(argv[ args + 1 ], "60")==0)
frame_size = 3*sampling_rate/50;
+ else if (strcmp(argv[ args + 1 ], "120")==0)
+ frame_size = 6*sampling_rate/50;
else {
fprintf(stderr, "Unsupported frame size: %s ms. "
- "Supported are 2.5, 5, 10, 20, 40, 60.\n",
+ "Supported are 2.5, 5, 10, 20, 40, 60, 120.\n",
argv[ args + 1 ]);
return EXIT_FAILURE;
}
@@ -610,8 +612,10 @@ int main(int argc, char *argv[])
variable_duration = OPUS_FRAMESIZE_20_MS;
else if (frame_size==sampling_rate/25)
variable_duration = OPUS_FRAMESIZE_40_MS;
- else
+ else if (frame_size==3*sampling_rate/50)
variable_duration = OPUS_FRAMESIZE_60_MS;
+ else
+ variable_duration = OPUS_FRAMESIZE_120_MS;
opus_encoder_ctl(enc, OPUS_SET_EXPERT_FRAME_DURATION(variable_duration));
}
frame_size = 2*48000;
diff --git a/src/opus_encoder.c b/src/opus_encoder.c
index 9305990..b004f3a 100644
--- a/src/opus_encoder.c
+++ b/src/opus_encoder.c
@@ -813,14 +813,14 @@ opus_int32 frame_size_select(opus_int32 frame_size, int variable_duration, opus_
new_size = frame_size;
else if (variable_duration == OPUS_FRAMESIZE_VARIABLE)
new_size = Fs/50;
- else if (variable_duration >= OPUS_FRAMESIZE_2_5_MS && variable_duration <= OPUS_FRAMESIZE_60_MS)
- new_size = IMIN(3*Fs/50, (Fs/400)<<(variable_duration-OPUS_FRAMESIZE_2_5_MS));
+ else if (variable_duration >= OPUS_FRAMESIZE_2_5_MS && variable_duration <= OPUS_FRAMESIZE_120_MS)
+ new_size = IMIN(6*Fs/50, (Fs/400)<<(variable_duration-OPUS_FRAMESIZE_2_5_MS));
else
return -1;
if (new_size>frame_size)
return -1;
if (400*new_size!=Fs && 200*new_size!=Fs && 100*new_size!=Fs &&
- 50*new_size!=Fs && 25*new_size!=Fs && 50*new_size!=3*Fs)
+ 50*new_size!=Fs && 25*new_size!=Fs && 50*new_size!=3*Fs && 25*new_size != 3*Fs)
return -1;
return new_size;
}
@@ -973,8 +973,8 @@ static opus_int32 encode_subframes_and_repacketize(OpusEncoder *st,
st->user_forced_mode = st->mode;
st->user_bandwidth = st->bandwidth;
st->force_channels = st->stream_channels;
- bak_to_mono = st->silk_mode.toMono;
+ bak_to_mono = st->silk_mode.toMono;
if (bak_to_mono)
st->force_channels = 1;
else
@@ -1010,7 +1010,7 @@ static opus_int32 encode_subframes_and_repacketize(OpusEncoder *st,
if (st->use_vbr)
repacketize_len = out_data_bytes;
else {
- cbr_bytes = st->bitrate_bps / (8 * st->Fs/frame_size);
+ cbr_bytes = st->bitrate_bps/(8*st->Fs/frame_size);
repacketize_len = IMIN(cbr_bytes*nb_frames, out_data_bytes);
}
@@ -1078,7 +1078,7 @@ opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_
st->rangeFinal = 0;
if ((!st->variable_duration && 400*frame_size != st->Fs && 200*frame_size != st->Fs && 100*frame_size != st->Fs &&
- 50*frame_size != st->Fs && 25*frame_size != st->Fs && 50*frame_size != 3*st->Fs)
+ 50*frame_size != st->Fs && 25*frame_size != st->Fs && 50*frame_size != 3*st->Fs && 25*frame_size != 3*st->Fs)
|| (400*frame_size < st->Fs)
|| max_data_bytes<=0
)
@@ -1445,15 +1445,32 @@ opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_
if (st->lfe)
st->bandwidth = OPUS_BANDWIDTH_NARROWBAND;
+ /* Handle 120 ms. Encode as 6x 20 ms subframes and then repacketize */
+ if (frame_size == 6*st->Fs/50)
+ {
+ int nb_subframes = 6;
+ int subframe_size = st->Fs/50;
+
+#ifndef DISABLE_FLOAT_API
+ if (analysis_read_pos_bak!= -1)
+ {
+ st->analysis.read_pos = analysis_read_pos_bak;
+ st->analysis.read_subframe = analysis_read_subframe_bak;
+ }
+#endif
+
+ ret = encode_subframes_and_repacketize(st, pcm, nb_subframes, subframe_size, data, out_data_bytes, to_celt,
+ lsb_depth, c1, c2, analysis_channels, downmix, float_api);
+
+ RESTORE_STACK;
+ return ret;
+ }
+
/* Can't support higher than wideband for >20 ms frames */
if (frame_size > st->Fs/50 && (st->mode == MODE_CELT_ONLY || st->bandwidth > OPUS_BANDWIDTH_WIDEBAND))
{
- int subframe_size;
- int nb_subframes;
-
- /* CELT can only support up to 20 ms */
- subframe_size = st->Fs/50;
- nb_subframes = frame_size > st->Fs/25 ? 3 : 2;
+ int subframe_size = st->Fs/50;
+ int nb_subframes = frame_size > st->Fs/25 ? 3 : 2;
#ifndef DISABLE_FLOAT_API
if (analysis_read_pos_bak!= -1)
@@ -2459,7 +2476,8 @@ int opus_encoder_ctl(OpusEncoder *st, int request, ...)
if (value != OPUS_FRAMESIZE_ARG && value != OPUS_FRAMESIZE_2_5_MS &&
value != OPUS_FRAMESIZE_5_MS && value != OPUS_FRAMESIZE_10_MS &&
value != OPUS_FRAMESIZE_20_MS && value != OPUS_FRAMESIZE_40_MS &&
- value != OPUS_FRAMESIZE_60_MS && value != OPUS_FRAMESIZE_VARIABLE)
+ value != OPUS_FRAMESIZE_60_MS && value != OPUS_FRAMESIZE_120_MS &&
+ value != OPUS_FRAMESIZE_VARIABLE)
{
goto bad_arg;
}
diff --git a/tests/test_opus_api.c b/tests/test_opus_api.c
index 9bfa5cc..53dfac0 100644
--- a/tests/test_opus_api.c
+++ b/tests/test_opus_api.c
@@ -1383,6 +1383,9 @@ opus_int32 test_enc_api(void)
err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_60_MS));
if(err!=OPUS_OK)test_failed();
cfgs++;
+ err=opus_encoder_ctl(enc,OPUS_SET_EXPERT_FRAME_DURATION(OPUS_FRAMESIZE_120_MS));
+ if(err!=OPUS_OK)test_failed();
+ cfgs++;
CHECK_SETGET(OPUS_SET_EXPERT_FRAME_DURATION(i),OPUS_GET_EXPERT_FRAME_DURATION(&i),0,-1,
OPUS_FRAMESIZE_60_MS,OPUS_FRAMESIZE_ARG,
" OPUS_SET_EXPERT_FRAME_DURATION ............... OK.\n",
--
2.8.0.rc3.226.g39d4020
-------------- next part --------------
From fc126bae557daf0e6cabf92806afe5815b4538ca Mon Sep 17 00:00:00 2001
From: Felicia Lim <flim at google.com>
Date: Tue, 31 May 2016 13:52:50 +0200
Subject: [PATCH 1/2] Refactor code for subframe encoding and repacketization
---
src/opus_encoder.c | 167 +++++++++++++++++++++++++++++++++--------------------
1 file changed, 105 insertions(+), 62 deletions(-)
diff --git a/src/opus_encoder.c b/src/opus_encoder.c
index e7b2f93..9305990 100644
--- a/src/opus_encoder.c
+++ b/src/opus_encoder.c
@@ -935,6 +935,103 @@ opus_val16 compute_stereo_width(const opus_val16 *pcm, int frame_size, opus_int3
return EXTRACT16(MIN32(Q15ONE, MULT16_16(20, mem->max_follower)));
}
+static opus_int32 encode_subframes_and_repacketize(OpusEncoder *st,
+ const opus_val16 *pcm,
+ int nb_frames,
+ int frame_size,
+ unsigned char *data,
+ opus_int32 out_data_bytes,
+ int to_celt,
+ int lsb_depth,
+ int c1,
+ int c2,
+ int analysis_channels,
+ downmix_func downmix,
+ int float_api)
+{
+ int i;
+ int ret = 0;
+ VARDECL(unsigned char, tmp_data);
+ int bak_mode, bak_bandwidth, bak_channels, bak_to_mono;
+ VARDECL(OpusRepacketizer, rp);
+ opus_int32 bytes_per_frame;
+ opus_int32 cbr_bytes;
+ opus_int32 repacketize_len;
+ int tmp_len;
+ ALLOC_STACK
+
+ bytes_per_frame = IMIN(1276,(out_data_bytes-3)/nb_frames);
+ ALLOC(tmp_data, nb_frames*bytes_per_frame, unsigned char);
+
+ ALLOC(rp, 1, OpusRepacketizer);
+ opus_repacketizer_init(rp);
+
+ bak_mode = st->user_forced_mode;
+ bak_bandwidth = st->user_bandwidth;
+ bak_channels = st->force_channels;
+
+ st->user_forced_mode = st->mode;
+ st->user_bandwidth = st->bandwidth;
+ st->force_channels = st->stream_channels;
+ bak_to_mono = st->silk_mode.toMono;
+
+ if (bak_to_mono)
+ st->force_channels = 1;
+ else
+ st->prev_channels = st->stream_channels;
+
+ for (i=0;i<nb_frames;i++)
+ {
+ st->silk_mode.toMono = 0;
+
+ /* When switching from SILK/Hybrid to CELT, only ask for a switch at the last frame */
+ if (to_celt && i==nb_frames-1)
+ st->user_forced_mode = MODE_CELT_ONLY;
+
+ tmp_len = opus_encode_native(st, pcm+i*(st->channels*frame_size), frame_size,
+ tmp_data+i*bytes_per_frame, bytes_per_frame, lsb_depth, NULL, 0, c1, c2, analysis_channels,
+ downmix, float_api);
+
+ if (tmp_len<0)
+ {
+ RESTORE_STACK;
+ return OPUS_INTERNAL_ERROR;
+ }
+
+ ret = opus_repacketizer_cat(rp, tmp_data+i*bytes_per_frame, tmp_len);
+
+ if (ret<0)
+ {
+ RESTORE_STACK;
+ return OPUS_INTERNAL_ERROR;
+ }
+ }
+
+ if (st->use_vbr)
+ repacketize_len = out_data_bytes;
+ else {
+ cbr_bytes = st->bitrate_bps / (8 * st->Fs/frame_size);
+ repacketize_len = IMIN(cbr_bytes*nb_frames, out_data_bytes);
+ }
+
+ ret = opus_repacketizer_out_range_impl(rp, 0, rp->nb_frames, data, repacketize_len, 0, !st->use_vbr);
+
+ if (ret<0)
+ {
+ RESTORE_STACK;
+ return OPUS_INTERNAL_ERROR;
+ }
+
+ /* Discard configs that were forced locally for the purpose of repacketization */
+ st->user_forced_mode = bak_mode;
+ st->user_bandwidth = bak_bandwidth;
+ st->force_channels = bak_channels;
+ st->silk_mode.toMono = bak_to_mono;
+
+ RESTORE_STACK;
+ return ret;
+}
+
opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_size,
unsigned char *data, opus_int32 out_data_bytes, int lsb_depth,
const void *analysis_pcm, opus_int32 analysis_size, int c1, int c2,
@@ -1351,12 +1448,12 @@ opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_
/* Can't support higher than wideband for >20 ms frames */
if (frame_size > st->Fs/50 && (st->mode == MODE_CELT_ONLY || st->bandwidth > OPUS_BANDWIDTH_WIDEBAND))
{
- VARDECL(unsigned char, tmp_data);
- int nb_frames;
- int bak_mode, bak_bandwidth, bak_channels, bak_to_mono;
- VARDECL(OpusRepacketizer, rp);
- opus_int32 bytes_per_frame;
- opus_int32 repacketize_len;
+ int subframe_size;
+ int nb_subframes;
+
+ /* CELT can only support up to 20 ms */
+ subframe_size = st->Fs/50;
+ nb_subframes = frame_size > st->Fs/25 ? 3 : 2;
#ifndef DISABLE_FLOAT_API
if (analysis_read_pos_bak!= -1)
@@ -1366,63 +1463,9 @@ opus_int32 opus_encode_native(OpusEncoder *st, const opus_val16 *pcm, int frame_
}
#endif
- nb_frames = frame_size > st->Fs/25 ? 3 : 2;
- bytes_per_frame = IMIN(1276,(out_data_bytes-3)/nb_frames);
-
- ALLOC(tmp_data, nb_frames*bytes_per_frame, unsigned char);
-
- ALLOC(rp, 1, OpusRepacketizer);
- opus_repacketizer_init(rp);
-
- bak_mode = st->user_forced_mode;
- bak_bandwidth = st->user_bandwidth;
- bak_channels = st->force_channels;
+ ret = encode_subframes_and_repacketize(st, pcm, nb_subframes, subframe_size, data, out_data_bytes, to_celt,
+ lsb_depth, c1, c2, analysis_channels, downmix, float_api);
- st->user_forced_mode = st->mode;
- st->user_bandwidth = st->bandwidth;
- st->force_channels = st->stream_channels;
- bak_to_mono = st->silk_mode.toMono;
-
- if (bak_to_mono)
- st->force_channels = 1;
- else
- st->prev_channels = st->stream_channels;
- for (i=0;i<nb_frames;i++)
- {
- int tmp_len;
- st->silk_mode.toMono = 0;
- /* When switching from SILK/Hybrid to CELT, only ask for a switch at the last frame */
- if (to_celt && i==nb_frames-1)
- st->user_forced_mode = MODE_CELT_ONLY;
- tmp_len = opus_encode_native(st, pcm+i*(st->channels*st->Fs/50), st->Fs/50,
- tmp_data+i*bytes_per_frame, bytes_per_frame, lsb_depth,
- NULL, 0, c1, c2, analysis_channels, downmix, float_api);
- if (tmp_len<0)
- {
- RESTORE_STACK;
- return OPUS_INTERNAL_ERROR;
- }
- ret = opus_repacketizer_cat(rp, tmp_data+i*bytes_per_frame, tmp_len);
- if (ret<0)
- {
- RESTORE_STACK;
- return OPUS_INTERNAL_ERROR;
- }
- }
- if (st->use_vbr)
- repacketize_len = out_data_bytes;
- else
- repacketize_len = IMIN(3*st->bitrate_bps/(3*8*50/nb_frames), out_data_bytes);
- ret = opus_repacketizer_out_range_impl(rp, 0, nb_frames, data, repacketize_len, 0, !st->use_vbr);
- if (ret<0)
- {
- RESTORE_STACK;
- return OPUS_INTERNAL_ERROR;
- }
- st->user_forced_mode = bak_mode;
- st->user_bandwidth = bak_bandwidth;
- st->force_channels = bak_channels;
- st->silk_mode.toMono = bak_to_mono;
RESTORE_STACK;
return ret;
}
--
2.8.0.rc3.226.g39d4020
More information about the opus
mailing list