From: Christian Beier <dontmind@freeshell.org>
Date: Sun, 6 Jan 2019 15:13:56 +0100
Subject: Error out in rfbProcessFileTransferReadBuffer if length can not be
 allocated
Origin: https://github.com/LibVNC/libvncserver/commit/15bb719c03cc70f14c36a843dcb16ed69b405707
Bug-Debian: https://bugs.debian.org/920941
Bug-Debian-Security: https://security-tracker.debian.org/tracker/CVE-2018-20749

re #273
---
 libvncserver/rfbserver.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

--- a/libvncserver/rfbserver.c
+++ b/libvncserver/rfbserver.c
@@ -1462,11 +1462,21 @@
     int   n=0;
 
     FILEXFER_ALLOWED_OR_CLOSE_AND_RETURN("", cl, NULL);
+
     /*
-    rfbLog("rfbProcessFileTransferReadBuffer(%dlen)\n", length);
+       We later alloc length+1, which might wrap around on 32-bit systems if length equals
+       0XFFFFFFFF, i.e. SIZE_MAX for 32-bit systems. On 64-bit systems, a length of 0XFFFFFFFF
+       will safely be allocated since this check will never trigger and malloc() can digest length+1
+       without problems as length is a uint32_t.
     */
+    if(length == SIZE_MAX) {
+	rfbErr("rfbProcessFileTransferReadBuffer: too big file transfer length requested: %u", (unsigned int)length);
+	rfbCloseClient(cl);
+	return NULL;
+    }
+
     if (length>0) {
-        buffer=malloc((uint64_t)length+1);
+        buffer=malloc((size_t)length+1);
         if (buffer!=NULL) {
             if ((n = rfbReadExact(cl, (char *)buffer, length)) <= 0) {
                 if (n != 0)
