[xiph-commits] r17253 - branches/lowmem-branch/Tremolo

[robin at svn.xiph.org]

Author: robin
Date: 2010-05-29 03:53:19 -0700 (Sat, 29 May 2010)
New Revision: 17253

Modified:
   branches/lowmem-branch/Tremolo/codebook.c
Log:
Address a stack blowout in _make_decode_table on WinCE.

test_genesis.ogg has entries=6561, and used_entries=81. This results in the
code attempting to alloca 54K on the stack. Instead change the code to use
malloc.




Modified: branches/lowmem-branch/Tremolo/codebook.c
===================================================================
--- branches/lowmem-branch/Tremolo/codebook.c	2010-05-29 03:46:41 UTC (rev 17252)
+++ branches/lowmem-branch/Tremolo/codebook.c	2010-05-29 10:53:19 UTC (rev 17253)
@@ -219,13 +219,16 @@
   if (s->used_entries > INT_MAX/2 ||
       s->used_entries*2 > INT_MAX/((long) sizeof(*work)) - 1) return 1;
   /* Overallocate as above */
-  work=alloca((s->entries*2+1)*sizeof(*work));
-  if(_make_words(lengthlist,s->entries,work,quantvals,s,opb,maptype))return 1;
-  if (s->used_entries > INT_MAX/(s->dec_leafw+1)) return 1;
-  if (s->dec_nodeb && s->used_entries * (s->dec_leafw+1) > INT_MAX/s->dec_nodeb) return 1;
+  /* With test_genesis.ogg, entries=6561, used_entries=81. Overallocating using
+   * alloca breaks the stack on WinCE, so use malloc instead. */
+  work=_ogg_malloc((s->entries*2+1)*sizeof(*work));
+  if (!work) return 1;
+  if(_make_words(lengthlist,s->entries,work,quantvals,s,opb,maptype)) goto fail_post_alloc;
+  if (s->used_entries > INT_MAX/(s->dec_leafw+1)) goto fail_post_alloc;
+  if (s->dec_nodeb && s->used_entries * (s->dec_leafw+1) > INT_MAX/s->dec_nodeb) goto fail_post_alloc;
   s->dec_table=_ogg_malloc((s->used_entries*(s->dec_leafw+1)-2)*
 			   s->dec_nodeb);
-  if (!s->dec_table) return 1;
+  if (!s->dec_table) goto fail_post_alloc;
   
   if(s->dec_leafw==1){
     switch(s->dec_nodeb){
@@ -310,6 +313,9 @@
   }
 	
   return 0;
+fail_post_alloc:
+  _ogg_free(work);
+  return 1;
 }
 
 /* most of the time, entries%dimensions == 0, but we need to be
@@ -473,9 +479,10 @@
     {
       /* packed values */
       long total1=(s->q_bits*s->dim+8)/8; /* remember flag bit */
+      long total2;
       if (s->dim > (INT_MAX-8)/s->q_bits) goto _eofout;
       /* vector of column offsets; remember flag bit */
-      long total2=(_ilog(quantvals-1)*s->dim+8)/8+(s->q_bits+7)/8;
+      total2=(_ilog(quantvals-1)*s->dim+8)/8+(s->q_bits+7)/8;
 
       
       if(total1<=4 && total1<=total2){