[Qemu-devel] [PATCH 6/7] Add tight protocol awareness to vnc.c

[Alexander Graf]

This patch enables the vnc server to understand the tight protocol.

Basically, negotiation if tight is existing happens through the
authentication code. If a special authentication called VNC_AUTH_TIGHT
is used, the real authentication is stacked afterwards and tight
extensions exist.

In order to make it easier to add different encodings later, this
patch also adds "preferred encoding" awareness, only saves the
last preferred encoding though.

This way the client can choose which protocol it wants to use the
most.

Signed-off-by: Alexander Graf <address@hidden>
---
 vnc.c |  122 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 files changed, 115 insertions(+), 7 deletions(-)

diff --git a/vnc.c b/vnc.c
index b4b0843..86e2a1b 100644
--- a/vnc.c
+++ b/vnc.c
@@ -103,6 +103,10 @@ struct VncState
     int last_x;
     int last_y;
 
+    int vnc_encoding;
+    unsigned int tight_quality;
+    unsigned int tight_compression;
+
     int major;
     int minor;
 
@@ -451,10 +455,14 @@ static void send_framebuffer_update_hextile(VncState *vs, 
int x, int y, int w, i
 
 static void send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
-       if (vnc_has_feature(vs, VNC_FEATURE_HEXTILE))
+    switch(vs->vnc_encoding) {
+       case VNC_ENCODING_HEXTILE:
            send_framebuffer_update_hextile(vs, x, y, w, h);
-       else
+           break;
+       default:
            send_framebuffer_update_raw(vs, x, y, w, h);
+           break;
+    }    
 }
 
 static void vnc_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int 
dst_y, int w, int h)
@@ -1165,19 +1173,38 @@ static void set_encodings(VncState *vs, int32_t 
*encodings, size_t n_encodings)
     unsigned int enc = 0;
 
     vs->features = 0;
+    vs->vnc_encoding = 0;
+    vs->tight_compression=0;
+    vs->tight_quality=9;
     vs->absolute = -1;
     dcl->dpy_copy = NULL;
 
     for (i = n_encodings - 1; i >= 0; i--) {
         enc = encodings[i];
         switch (enc) {
+        case VNC_ENCODING_RRE:
+        case VNC_ENCODING_CORRE:
+        case VNC_ENCODING_ZLIB:
+        case VNC_ENCODING_ZLIBHEX:
+        case VNC_ENCODING_ZRLE:
+        case VNC_ENCODING_LASTRECT:
+        case VNC_ENCODING_POINTER_POS:
+        case VNC_ENCODING_RICH_CURSOR:
+        case VNC_ENCODING_XCURSOR:
+            /* Ingore - at least for now... */
+            break;
         case VNC_ENCODING_RAW:
+            vs->vnc_encoding = enc;
             break;
         case VNC_ENCODING_COPYRECT:
             dcl->dpy_copy = vnc_copy;
             break;
         case VNC_ENCODING_HEXTILE:
             vs->features |= VNC_FEATURE_HEXTILE_MASK;
+            vs->vnc_encoding = enc;
+            break;
+        case VNC_ENCODING_TIGHT:
+            vs->features |= VNC_FEATURE_TIGHT_MASK;
             break;
         case VNC_ENCODING_DESKTOPRESIZE:
             vs->features |= VNC_FEATURE_RESIZE_MASK;
@@ -1194,6 +1221,12 @@ static void set_encodings(VncState *vs, int32_t 
*encodings, size_t n_encodings)
         case VNC_ENCODING_WMVi:
             vs->features |= VNC_FEATURE_WMVI_MASK;
             break;
+        case VNC_ENCODING_COMPRESSLEVEL0 ... VNC_ENCODING_COMPRESSLEVEL0 + 9:
+            vs->tight_compression = (enc & 0x0F);
+            break;
+        case VNC_ENCODING_QUALITYLEVEL0 ... VNC_ENCODING_QUALITYLEVEL0 + 9:
+            vs->tight_quality = (enc & 0x0F);
+            break;
         default:
              printf("Unknown encoding: %d (0x%.8x): %d\n", i, enc, enc);
             break;
@@ -1461,6 +1494,36 @@ static int protocol_client_msg(VncState *vs, uint8_t 
*data, size_t len)
     return 0;
 }
 
+static void write_tight_capability(VncState *vs, int encoding,
+                                   const char *signature, const char *vendor)
+{
+       vnc_write_u32(vs, encoding);
+       vnc_write(vs, vendor, 4);
+       vnc_write(vs, signature, 8);
+}
+
+static void write_tight_interaction_capabilities(VncState *vs)
+{
+    VNC_DEBUG("Sending Tight capabilities...\n");
+    vnc_write_u16(vs, 0); /* nServerMessageTypes  */
+    vnc_write_u16(vs, 0); /* nClientMessageTypes  */ 
+    vnc_write_u16(vs, 5); /* nEncodingTypes */
+    vnc_write_u16(vs, 0); /* pad */
+    write_tight_capability(vs, VNC_ENCODING_TIGHT, VNC_ENCODING_TIGHT_SIG,
+                           VNC_VENDOR_TIGHT);
+    write_tight_capability(vs, VNC_ENCODING_RAW, VNC_ENCODING_RAW_SIG,
+                           VNC_VENDOR_STANDARD);
+    write_tight_capability(vs, VNC_ENCODING_HEXTILE, VNC_ENCODING_HEXTILE_SIG,
+                           VNC_VENDOR_STANDARD);
+    write_tight_capability(vs, VNC_ENCODING_COMPRESSLEVEL0,
+                           VNC_ENCODING_COMPRESSLEVEL0_SIG, VNC_VENDOR_TIGHT);
+    write_tight_capability(vs, VNC_ENCODING_QUALITYLEVEL0,
+                           VNC_ENCODING_QUALITYLEVEL0_SIG, VNC_VENDOR_TIGHT);
+
+    vnc_flush(vs);
+}
+
+
 static int protocol_client_init(VncState *vs, uint8_t *data, size_t len)
 {
     char buf[1024];
@@ -1480,6 +1543,9 @@ static int protocol_client_init(VncState *vs, uint8_t 
*data, size_t len)
     vnc_write(vs, buf, size);
     vnc_flush(vs);
 
+    if(vnc_has_feature(vs, VNC_FEATURE_TIGHT_PROTOCOL))
+        write_tight_interaction_capabilities(vs);
+    
     vnc_read_when(vs, protocol_client_msg, 1);
 
     return 0;
@@ -1556,6 +1622,32 @@ static int start_auth_vnc(VncState *vs)
     return 0;
 }
 
+static int start_auth_tight(VncState *vs)
+{
+    vnc_write_u32(vs, 0); /* no tunnel types */
+    vnc_write_u32(vs, 1); /* num auth types */
+        
+    switch(vs->auth) {
+        case VNC_AUTH_NONE:
+            write_tight_capability(vs, VNC_AUTH_NONE, VNC_AUTH_NONE_SIG,
+                                   VNC_VENDOR_STANDARD);
+            break;
+        case VNC_AUTH_VNC:
+            write_tight_capability(vs, VNC_AUTH_VNC, VNC_AUTH_VNC_SIG,
+                                   VNC_VENDOR_STANDARD);
+            break;
+        default:
+            VNC_DEBUG("Unknown subset of tight auth: %d", vs->auth);
+            vnc_write_u32(vs, -1);
+            vnc_write(vs, "BAD_", 4);
+            vnc_write(vs, "UNKNOWN_", 8);
+    }
+    
+    vnc_flush(vs);
+    vnc_read_when(vs, protocol_client_auth, 4);
+    return 0;
+}
+
 static int vnc_start_auth(VncState *vs, int auth)
 {
     switch (auth) {
@@ -1570,12 +1662,22 @@ static int vnc_start_auth(VncState *vs, int auth)
     case VNC_AUTH_VNC:
         VNC_DEBUG("Start VNC auth\n");
         return start_auth_vnc(vs);
+    case VNC_AUTH_TIGHT:
+        /* If we know we're on tight here, we ran into recursion */
+        if (vnc_has_feature(vs, VNC_FEATURE_TIGHT_PROTOCOL)) {
+            VNC_DEBUG("Danger! Server sent us tight auth request twice!\n");
+            goto tight_recursion;
+        }
+        vs->features |= VNC_FEATURE_TIGHT_PROTOCOL_MASK;
+        VNC_DEBUG("Start VNC tight auth\n");
+        return start_auth_tight(vs);
 #ifdef CONFIG_VNC_TLS
     case VNC_AUTH_VENCRYPT:
         VNC_DEBUG("Accept VeNCrypt auth\n");;
         return start_auth_vencrypt(vs);
 #endif /* CONFIG_VNC_TLS */
     default: /* Should not be possible, but just in case */
+    tight_recursion:
         VNC_DEBUG("Reject auth %d unknown\n", auth);
         vnc_write_u8(vs, 1);
         if (vs->minor >= 8) {
@@ -2001,10 +2103,15 @@ static int start_auth_vencrypt(VncState *vs)
 
 static int protocol_client_auth(VncState *vs, uint8_t *data, size_t len)
 {
+    int auth = data[0];
+
+    if (len == 4)
+        auth = read_s32(data, 0);
     /* We only advertise 1 auth scheme at a time, so client
      * must pick the one we sent. Verify this */
-    if (data[0] != vs->auth) { /* Reject auth */
-       VNC_DEBUG("Reject auth %d\n", (int)data[0]);
+    if ((auth != vs->auth) &&
+        (auth != VNC_AUTH_TIGHT)) { /* Reject auth */
+       VNC_DEBUG("Reject auth %d != %d\n", auth, vs->auth);
        vnc_write_u32(vs, 1);
        if (vs->minor >= 8) {
            static const char err[] = "Authentication failed";
@@ -2014,7 +2121,7 @@ static int protocol_client_auth(VncState *vs, uint8_t 
*data, size_t len)
        vnc_client_error(vs);
     } else { /* Accept requested auth */
        VNC_DEBUG("Client requested auth %d\n", auth);
-       return vnc_start_auth(vs, data[0]);
+       return vnc_start_auth(vs, auth);
     }
     return 0;
 }
@@ -2068,9 +2175,10 @@ static int protocol_version(VncState *vs, uint8_t 
*version, size_t len)
             vnc_client_error(vs);
        }
     } else {
-       VNC_DEBUG("Telling client we support auth %d\n", vs->auth);
-       vnc_write_u8(vs, 1); /* num auth */
+       VNC_DEBUG("Telling client we support auth %d and tight\n", vs->auth);
+       vnc_write_u8(vs, 2); /* num auth */
        vnc_write_u8(vs, vs->auth);
+       vnc_write_u8(vs, VNC_AUTH_TIGHT);
        vnc_read_when(vs, protocol_client_auth, 1);
        vnc_flush(vs);
     }
-- 
1.6.0.2