[Qemu-devel] [6413] fix endianness problem sharing the videoram buffer

[malc]

Revision: 6413
          http://svn.sv.gnu.org/viewvc/?view=rev&root=qemu&revision=6413
Author:   malc
Date:     2009-01-23 19:56:19 +0000 (Fri, 23 Jan 2009)

Log Message:
-----------
fix endianness problem sharing the videoram buffer

[ The following text is in the "UTF-8" character set. ]

    [ Your display is set for the "koi8-r" character set.  ]

    [ Some characters may be displayed incorrectly. ]

This patch fixes vga rendering when the guest endianness differs from
the host endianness: in this case we can only share the buffer if the
bpp is 32 and we must change the pixelformat accordingly.

Signed-off-by: Stefano Stabellini <address@hidden>

Modified Paths:
--------------
    trunk/console.c
    trunk/console.h
    trunk/hw/vga.c

Modified: trunk/console.c
===================================================================
--- trunk/console.c     2009-01-23 15:02:20 UTC (rev 6412)
+++ trunk/console.c     2009-01-23 19:56:19 UTC (rev 6413)
@@ -1449,7 +1449,7 @@
     }
 }
 
-static PixelFormat qemu_default_pixelformat(int bpp)
+PixelFormat qemu_different_endianness_pixelformat(int bpp)
 {
     PixelFormat pf;
 
@@ -1460,17 +1460,48 @@
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
-        case 8:
-            pf.rmask = 0x000000E0;
-            pf.gmask = 0x0000001C;
-            pf.bmask = 0x00000003;
-            pf.rmax = 7;
-            pf.gmax = 7;
-            pf.bmax = 3;
-            pf.rshift = 5;
-            pf.gshift = 2;
-            pf.bshift = 0;
+        case 24:
+            pf.rmask = 0x000000FF;
+            pf.gmask = 0x0000FF00;
+            pf.bmask = 0x00FF0000;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.rshift = 0;
+            pf.gshift = 8;
+            pf.bshift = 16;
             break;
+        case 32:
+            pf.rmask = 0x0000FF00;
+            pf.gmask = 0x00FF0000;
+            pf.bmask = 0xFF000000;
+            pf.amask = 0x00000000;
+            pf.amax = 255;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.ashift = 0;
+            pf.rshift = 8;
+            pf.gshift = 16;
+            pf.bshift = 24;
+            break;
+        default:
+            break;
+    }
+    return pf;
+}
+
+PixelFormat qemu_default_pixelformat(int bpp)
+{
+    PixelFormat pf;
+
+    memset(&pf, 0x00, sizeof(PixelFormat));
+
+    pf.bits_per_pixel = bpp;
+    pf.bytes_per_pixel = bpp / 8;
+    pf.depth = bpp == 32 ? 24 : bpp;
+
+    switch (bpp) {
         case 16:
             pf.rmask = 0x0000F800;
             pf.gmask = 0x000007E0;
@@ -1483,13 +1514,24 @@
             pf.bshift = 0;
             break;
         case 24:
+            pf.rmask = 0x00FF0000;
+            pf.gmask = 0x0000FF00;
+            pf.bmask = 0x000000FF;
+            pf.rmax = 255;
+            pf.gmax = 255;
+            pf.bmax = 255;
+            pf.rshift = 16;
+            pf.gshift = 8;
+            pf.bshift = 0;
         case 32:
             pf.rmask = 0x00FF0000;
             pf.gmask = 0x0000FF00;
             pf.bmask = 0x000000FF;
+            pf.amax = 255;
             pf.rmax = 255;
             pf.gmax = 255;
             pf.bmax = 255;
+            pf.ashift = 24;
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;

Modified: trunk/console.h
===================================================================
--- trunk/console.h     2009-01-23 15:02:20 UTC (rev 6412)
+++ trunk/console.h     2009-01-23 19:56:19 UTC (rev 6413)
@@ -134,6 +134,8 @@
 DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
                                                 int linesize, uint8_t *data);
 void qemu_free_displaysurface(DisplaySurface *surface);
+PixelFormat qemu_different_endianness_pixelformat(int bpp);
+PixelFormat qemu_default_pixelformat(int bpp);
 
 static inline int is_buffer_shared(DisplaySurface *surface)
 {

Modified: trunk/hw/vga.c
===================================================================
--- trunk/hw/vga.c      2009-01-23 15:02:20 UTC (rev 6412)
+++ trunk/hw/vga.c      2009-01-23 19:56:19 UTC (rev 6413)
@@ -1623,12 +1623,19 @@
         disp_width != s->last_width ||
         height != s->last_height ||
         s->last_depth != depth) {
+#if defined(WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
         if (depth == 16 || depth == 32) {
+#else
+        if (depth == 32) {
+#endif
             if (is_graphic_console()) {
                 qemu_free_displaysurface(s->ds->surface);
                 s->ds->surface = qemu_create_displaysurface_from(disp_width, 
height, depth,
                                                                s->line_offset,
                                                                s->vram_ptr + 
(s->start_addr * 4));
+#if defined(WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
+                s->ds->surface->pf = 
qemu_different_endianness_pixelformat(depth);
+#endif
                 dpy_resize(s->ds);
             } else {
                 qemu_console_resize(s->ds, disp_width, height);