$OpenBSD: patch-tools_tiff2rgba_c,v 1.1 2009/07/22 13:15:00 jasper Exp $

Fix several places in tiff2rgba and rgb2ycbcr that were being careless about
possible integer overflow in calculation of buffer sizes.

CVE-2009-2347

--- tools/tiff2rgba.c.orig	Sun Nov  7 12:08:37 2004
+++ tools/tiff2rgba.c	Wed Jul 22 13:41:02 2009
@@ -124,6 +124,17 @@ main(int argc, char* argv[])
     return (0);
 }
 
+static tsize_t
+multiply(tsize_t m1, tsize_t m2)
+{
+    tsize_t prod = m1 * m2;
+
+    if (m1 && prod / m1 != m2)
+        prod = 0;		/* overflow */
+
+    return prod;
+}
+
 static int
 cvt_by_tile( TIFF *in, TIFF *out )
 
@@ -133,6 +144,7 @@ cvt_by_tile( TIFF *in, TIFF *out )
     uint32  tile_width, tile_height;
     uint32  row, col;
     uint32  *wrk_line;
+    tsize_t raster_size;
     int	    ok = 1;
 
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
@@ -150,7 +162,14 @@ cvt_by_tile( TIFF *in, TIFF *out )
     /*
      * Allocate tile buffer
      */
-    raster = (uint32*)_TIFFmalloc(tile_width * tile_height * sizeof (uint32));
+    raster_size = multiply(multiply(tile_width, tile_height), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) tile_width, (unsigned long) tile_height);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -158,7 +177,7 @@ cvt_by_tile( TIFF *in, TIFF *out )
 
     /*
      * Allocate a scanline buffer for swapping during the vertical
-     * mirroring pass.
+     * mirroring pass.  (Request can't overflow given prior checks.)
      */
     wrk_line = (uint32*)_TIFFmalloc(tile_width * sizeof (uint32));
     if (!wrk_line) {
@@ -226,6 +245,7 @@ cvt_by_strip( TIFF *in, TIFF *out )
     uint32  width, height;		/* image width & height */
     uint32  row;
     uint32  *wrk_line;
+    tsize_t raster_size;
     int	    ok = 1;
 
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
@@ -241,7 +261,14 @@ cvt_by_strip( TIFF *in, TIFF *out )
     /*
      * Allocate strip buffer
      */
-    raster = (uint32*)_TIFFmalloc(width * rowsperstrip * sizeof (uint32));
+    raster_size = multiply(multiply(width, rowsperstrip), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) width, (unsigned long) rowsperstrip);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -249,7 +276,7 @@ cvt_by_strip( TIFF *in, TIFF *out )
 
     /*
      * Allocate a scanline buffer for swapping during the vertical
-     * mirroring pass.
+     * mirroring pass.  (Request can't overflow given prior checks.)
      */
     wrk_line = (uint32*)_TIFFmalloc(width * sizeof (uint32));
     if (!wrk_line) {
@@ -328,14 +355,22 @@ cvt_whole_image( TIFF *in, TIFF *out )
     uint32* raster;			/* retrieve RGBA image */
     uint32  width, height;		/* image width & height */
     uint32  row;
-        
+    tsize_t raster_size;
+
     TIFFGetField(in, TIFFTAG_IMAGEWIDTH, &width);
     TIFFGetField(in, TIFFTAG_IMAGELENGTH, &height);
 
     rowsperstrip = TIFFDefaultStripSize(out, rowsperstrip);
     TIFFSetField(out, TIFFTAG_ROWSPERSTRIP, rowsperstrip);
 
-    raster = (uint32*)_TIFFmalloc(width * height * sizeof (uint32));
+    raster_size = multiply(multiply(width, height), sizeof (uint32));
+    if (!raster_size) {
+	TIFFError(TIFFFileName(in),
+		  "Can't allocate buffer for raster of size %lux%lu",
+		  (unsigned long) width, (unsigned long) height);
+	return (0);
+    }
+    raster = (uint32*)_TIFFmalloc(raster_size);
     if (raster == 0) {
         TIFFError(TIFFFileName(in), "No space for raster buffer");
         return (0);
@@ -353,7 +388,7 @@ cvt_whole_image( TIFF *in, TIFF *out )
     */
     if( no_alpha )
     {
-        int	pixel_count = width * height;
+        tsize_t  pixel_count = (tsize_t) width * (tsize_t) height;
         unsigned char *src, *dst;
 
         src = (unsigned char *) raster;
