Patch-Mainline: To be upstreamed
References: bnc#900896 CVE-2014-8240
Signed-off-by: Michal Srb <msrb@suse.com>

diff -up tigervnc-1.3.1/unix/x0vncserver/Image.cxx.CVE-2014-8240 tigervnc-1.3.1/unix/x0vncserver/Image.cxx
--- tigervnc-1.3.1/unix/x0vncserver/Image.cxx.CVE-2014-8240	2008-03-19 16:14:48.000000000 +0000
+++ tigervnc-1.3.1/unix/x0vncserver/Image.cxx	2014-10-16 12:23:08.013339234 +0100
@@ -80,6 +80,14 @@ void Image::Init(int width, int height)
   xim = XCreateImage(dpy, vis, DefaultDepth(dpy, DefaultScreen(dpy)),
                      ZPixmap, 0, 0, width, height, BitmapPad(dpy), 0);
 
+  if (xim->bytes_per_line <= 0 ||
+      xim->height <= 0 ||
+      xim->height >= INT_MAX / xim->bytes_per_line) {
+    vlog.error("Invalid display size");
+    XDestroyImage(xim);
+    exit(1);
+  }
+
   xim->data = (char *)malloc(xim->bytes_per_line * xim->height);
   if (xim->data == NULL) {
     vlog.error("malloc() failed");
@@ -254,6 +262,17 @@ void ShmImage::Init(int width, int heigh
     delete shminfo;
     shminfo = NULL;
     return;
+  }
+
+  if (xim->bytes_per_line <= 0 ||
+      xim->height <= 0 ||
+      xim->height >= INT_MAX / xim->bytes_per_line) {
+    vlog.error("Invalid display size");
+    XDestroyImage(xim);
+    xim = NULL;
+    delete shminfo;
+    shminfo = NULL;
+    return;
   }
 
   shminfo->shmid = shmget(IPC_PRIVATE,
diff -up tigervnc-1.3.1/vncviewer/X11PixelBuffer.cxx.CVE-2014-8240 tigervnc-1.3.1/vncviewer/X11PixelBuffer.cxx
--- tigervnc-1.3.1/vncviewer/X11PixelBuffer.cxx.CVE-2014-8240	2011-08-23 13:04:46.000000000 +0100
+++ tigervnc-1.3.1/vncviewer/X11PixelBuffer.cxx	2014-10-16 12:22:53.053261132 +0100
@@ -105,6 +105,15 @@ PlatformPixelBuffer::PlatformPixelBuffer
                        ZPixmap, 0, 0, width, height, BitmapPad(fl_display), 0);
     assert(xim);
 
+    if (xim->bytes_per_line <= 0 ||
+	xim->height <= 0 ||
+	xim->height >= INT_MAX / xim->bytes_per_line) {
+      if (xim)
+	XDestroyImage(xim);
+      xim = NULL;
+      throw rfb::Exception("Invalid display size");
+    }
+
     xim->data = (char*)malloc(xim->bytes_per_line * xim->height);
     assert(xim->data);
   }
@@ -169,6 +178,16 @@ int PlatformPixelBuffer::setupShm()
   if (!xim)
     goto free_shminfo;
 
+  if (xim->bytes_per_line <= 0 ||
+      xim->height <= 0 ||
+      xim->height >= INT_MAX / xim->bytes_per_line) {
+    XDestroyImage(xim);
+    xim = NULL;
+    delete shminfo;
+    shminfo = NULL;
+    throw rfb::Exception("Invalid display size");
+  }
+
   shminfo->shmid = shmget(IPC_PRIVATE,
                           xim->bytes_per_line * xim->height,
                           IPC_CREAT|0777);