[xiph-commits] r17270 - in trunk/ogg: include/ogg src

[xiphmont at svn.xiph.org]

Author: xiphmont
Date: 2010-06-03 23:01:33 -0700 (Thu, 03 Jun 2010)
New Revision: 17270

Modified:
   trunk/ogg/include/ogg/os_types.h
   trunk/ogg/src/bitwise.c
Log:
Two cleanups of buffer LONG_MAX overflow hardening:

GCC optimizes out the overflow check due to the overflow check reyling on overflow; 
reimplement using type-based TYPE_MAX macro

Correct an accidental assignment-during-check that wasn't a bug, but was 
semantically incorrect and rightly triggered a compilation warning.



Modified: trunk/ogg/include/ogg/os_types.h
===================================================================
--- trunk/ogg/include/ogg/os_types.h	2010-06-04 05:39:45 UTC (rev 17269)
+++ trunk/ogg/include/ogg/os_types.h	2010-06-04 06:01:33 UTC (rev 17270)
@@ -24,6 +24,19 @@
 #define _ogg_realloc realloc
 #define _ogg_free    free
 
+/* get non-brittle portable type-based MIN/MAX. Assumes 2's-complement
+   math */
+#define TYPE_HALF_MAX_SIGNED(type) \
+  ((typeof(type))1 << (sizeof(type)*8-2))
+#define TYPE_MAX_SIGNED(type) \
+  (TYPE_HALF_MAX_SIGNED(type) - 1 + TYPE_HALF_MAX_SIGNED(type))
+#define TYPE_MIN_SIGNED(type) \
+  (-1 - TYPE_MAX_SIGNED(type))
+#define TYPE_MIN(type) \
+  ((typeof(type))-1 < 1?TYPE_MIN_SIGNED(type):(typeof(type))0)
+#define TYPE_MAX(type) \
+  ((typeof(type))~TYPE_MIN(type))
+
 #if defined(_WIN32) 
 
 #  if defined(__CYGWIN__)

Modified: trunk/ogg/src/bitwise.c
===================================================================
--- trunk/ogg/src/bitwise.c	2010-06-04 05:39:45 UTC (rev 17269)
+++ trunk/ogg/src/bitwise.c	2010-06-04 06:01:33 UTC (rev 17270)
@@ -84,7 +84,7 @@
   if(b->endbyte>=b->storage-4){
     void *ret;
     if(!b->ptr)return;
-    if(b->storage+BUFFER_INCREMENT<b->storage) goto err;
+    if(b->storage>TYPE_MAX(b->storage)-BUFFER_INCREMENT) goto err;
     ret=_ogg_realloc(b->buffer,b->storage+BUFFER_INCREMENT);
     if(!ret) goto err;
     b->buffer=ret;
@@ -127,7 +127,7 @@
   if(b->endbyte>=b->storage-4){
     void *ret;
     if(!b->ptr)return;
-    if(b->storage+BUFFER_INCREMENT<b->storage) goto err;
+    if(b->storage>TYPE_MAX(b->storage)-BUFFER_INCREMENT) goto err;
     ret=_ogg_realloc(b->buffer,b->storage+BUFFER_INCREMENT);
     if(!ret) goto err;
     b->buffer=ret;
@@ -198,7 +198,7 @@
     if(b->endbyte+bytes+1>=b->storage){
       void *ret;
       if(!b->ptr) goto err;
-      if(b->storage=b->endbyte+bytes+BUFFER_INCREMENT>b->storage) goto err;
+      if(b->endbyte+bytes+BUFFER_INCREMENT>b->storage) goto err;
       b->storage=b->endbyte+bytes+BUFFER_INCREMENT;
       ret=_ogg_realloc(b->buffer,b->storage);
       if(!ret) goto err;