[Qemu-devel] [PATCH v4] hw/display/qxl: fix signed to unsigned comparison

[Alon Levy]

Several small signedness / overflow corrections to qxl_create_guest_primary:
1. use 64 bit unsigned for size to avoid overflow possible from two 32
bit multiplicants.
2. correct sign for requested_height
3. add a more verbose error message when setting guest bug state (which
causes a complete guess blackout until reset, so it helps if it is
verbose).

Signed-off-by: Alon Levy <address@hidden>
---
 hw/display/qxl.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index e4f172e..ceae1d9 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -19,6 +19,7 @@
  */
 
 #include <zlib.h>
+#include <stdint.h>
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
@@ -1360,14 +1361,16 @@ static void qxl_create_guest_primary(PCIQXLDevice *qxl, 
int loadvm,
 {
     QXLDevSurfaceCreate surface;
     QXLSurfaceCreate *sc = &qxl->guest_primary.surface;
-    int size;
-    int requested_height = le32_to_cpu(sc->height);
+    uint32_t requested_height = le32_to_cpu(sc->height);
     int requested_stride = le32_to_cpu(sc->stride);
 
-    size = abs(requested_stride) * requested_height;
-    if (size > qxl->vgamem_size) {
-        qxl_set_guest_bug(qxl, "%s: requested primary larger then framebuffer"
-                               " size", __func__);
+    if (requested_stride == INT32_MIN ||
+        abs(requested_stride) * (uint64_t)requested_height
+                                        > qxl->vgamem_size) {
+        qxl_set_guest_bug(qxl, "%s: requested primary larger than framebuffer"
+                               " stride %d x height %" PRIu32 " > %" PRIu32,
+                               __func__, requested_stride, requested_height,
+                               qxl->vgamem_size);
         return;
     }
 
-- 
1.8.4.2