[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] [PULL 3/3] vnc-tight: fix regression with libxenstore
From: |
Gerd Hoffmann |
Subject: |
[Qemu-devel] [PULL 3/3] vnc-tight: fix regression with libxenstore |
Date: |
Tue, 19 Jul 2016 09:28:18 +0200 |
From: Peter Lieven <address@hidden>
commit 095497ff added thread local storage for the color counting
palette. Unfortunately, a VncPalette is about 7kB on a x86_64 system.
This memory is reserved from the stack of every thread and it
exhausted the stack space of a libxenstore thread.
Fix this by allocating memory only for the VNC encoding thread.
Fixes: 095497ffc66b7f031ff2a17f1e50f5cb105ce588
Reported-by: Juergen Gross <address@hidden>
Tested-by: Juergen Gross <address@hidden>
Signed-off-by: Peter Lieven <address@hidden>
Message-id: address@hidden
Signed-off-by: Gerd Hoffmann <address@hidden>
---
ui/vnc-enc-tight.c | 28 +++++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 877c093..49df85e 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -1458,11 +1458,17 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int
y, int w, int h,
}
#endif
-static __thread VncPalette color_count_palette;
+static __thread VncPalette *color_count_palette;
+static __thread Notifier vnc_tight_cleanup_notifier;
+
+static void vnc_tight_cleanup(Notifier *n, void *value)
+{
+ g_free(color_count_palette);
+ color_count_palette = NULL;
+}
static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
{
- VncPalette *palette = &color_count_palette;
uint32_t bg = 0, fg = 0;
int colors;
int ret = 0;
@@ -1471,6 +1477,12 @@ static int send_sub_rect(VncState *vs, int x, int y, int
w, int h)
bool allow_jpeg = true;
#endif
+ if (!color_count_palette) {
+ color_count_palette = g_malloc(sizeof(VncPalette));
+ vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup;
+ qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
+ }
+
vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
vnc_tight_start(vs);
@@ -1491,17 +1503,19 @@ static int send_sub_rect(VncState *vs, int x, int y,
int w, int h)
}
#endif
- colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, palette);
+ colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg,
color_count_palette);
#ifdef CONFIG_VNC_JPEG
if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
- ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette,
- force_jpeg);
+ ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
+ color_count_palette, force_jpeg);
} else {
- ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+ ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+ color_count_palette);
}
#else
- ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+ ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+ color_count_palette);
#endif
return ret;
--
1.8.3.1