[Qemu-devel] [PATCH] Add HTTP protocol using curl v2

[alex]

From: Alexander Graf <address@hidden>

Currently Qemu can read from posix I/O and NBD. This patch adds a
third protocol to the game: HTTP.

In certain situations it can be useful to access HTTP data directly,
for example if you want to try out an http provided OS image, but
don't know if you want to download it yet.

Using this patch you can now try it on on the fly. Just use it like:

qemu -cdrom http://host/path/my.iso

In order to not reinvent the wheel, this patch uses libcurl.

v2 changes:

- fix the segfault (yay!)
- implement AIO

I am still missing caching completely, so if you're accessing a WAN server
it'll be rather slow. Also, a lot of small partial accesses will clutter
the server's logs which is not ideal.

I'm open to suggestions on how to implement caching. I was thinking of a model
similar to -snapshot, but invoked from within the http backend, with the
option to write the file to a real file too, so you can use qemu as download
application while already using the image.

I haven't thought out resuming downloads for that one yet though :-/. Maybe
scanning the download file for sparse spots and force download those chunks?

Again, I'm open to suggestions :-). The patch as is is rather useful already
though as long as you stay in a LAN (or have a really fast internet connection).

Also please give me ideas on how and where to put this feature in the docs.

Signed-off-by: Alexander Graf <address@hidden>
---
 Makefile        |    6 +
 Makefile.target |    2 +-
 block-http.c    |  381 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 block.c         |    3 +
 block.h         |    1 +
 configure       |   25 ++++
 6 files changed, 417 insertions(+), 1 deletions(-)
 create mode 100644 block-http.c

diff --git a/Makefile b/Makefile
index 3c70068..fbfbc6e 100644
--- a/Makefile
+++ b/Makefile
@@ -77,6 +77,10 @@ endif
 BLOCK_OBJS += block-raw-posix.o
 endif
 
+ifdef CONFIG_CURL
+BLOCK_OBJS += block-http.o
+endif
+
 ######################################################################
 # libqemu_common.a: Target independent part of system emulation. The
 # long term path is to suppress *all* target specific code in case of
@@ -185,6 +189,8 @@ endif
 
 LIBS+=$(VDE_LIBS)
 
+LIBS+=$(CURL_LIBS)
+
 cocoa.o: cocoa.m
 
 keymaps.o: keymaps.c keymaps.h
diff --git a/Makefile.target b/Makefile.target
index f735105..3a9226c 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -735,7 +735,7 @@ endif
 
 vl.o: qemu-options.h
 
-$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) 
$(VDE_LIBS)
+$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS) 
$(VDE_LIBS) $(CURL_LIBS)
 
 $(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a
        $(LINK)
diff --git a/block-http.c b/block-http.c
new file mode 100644
index 0000000..9842b20
--- /dev/null
+++ b/block-http.c
@@ -0,0 +1,381 @@
+/*
+ * QEMU Block driver for HTTP images
+ *
+ * Copyright (c) 2009 Alexander Graf <address@hidden>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to 
deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block_int.h"
+#include <curl/curl.h>
+
+// #define DEBUG
+// #define DEBUG_VERBOSE
+
+#ifdef DEBUG
+#define printd printf
+#else
+#define printd(x, ...)
+#endif
+
+#ifdef CONFIG_AIO
+#define HTTP_NUM_STATES 8
+#else
+#define HTTP_NUM_STATES 1
+#endif
+
+#define SECTOR_SIZE 512
+
+struct BDRVHTTPState;
+
+#ifdef CONFIG_AIO
+typedef struct HTTPAIOCB {
+    BlockDriverAIOCB common;
+    QEMUIOVector *qiov;
+} HTTPAIOCB;
+#endif
+
+typedef struct HTTPState
+{
+    struct BDRVHTTPState *s;
+    CURL *curl;
+    uint8_t *buf;
+    int64_t buf_len;
+    char range[128];
+    char errmsg[CURL_ERROR_SIZE];
+    char in_use;
+#ifdef CONFIG_AIO
+    uint8_t *orig_buf;
+    HTTPAIOCB *acb;
+#endif
+} HTTPState;
+
+typedef struct BDRVHTTPState {
+    CURLM *multi;
+    int64_t len;
+    HTTPState states[HTTP_NUM_STATES];
+    char *url;
+} BDRVHTTPState;
+
+static void http_clean_state(HTTPState *s);
+static void http_multi_do(void *arg);
+
+#ifdef CONFIG_AIO
+static int http_sock_cb(CURL *curl, curl_socket_t fd, int action,
+                        void *s, void *sp)
+{
+    printd("CURL (AIO): Sock action %d on fd %d\n", action, fd);
+    switch (action) {
+        case CURL_POLL_IN:
+            qemu_aio_set_fd_handler(fd, http_multi_do, NULL, NULL, s);
+            break;
+        case CURL_POLL_OUT:
+            qemu_aio_set_fd_handler(fd, NULL, http_multi_do, NULL, s);
+            break;
+        case CURL_POLL_INOUT:
+            qemu_aio_set_fd_handler(fd, http_multi_do,
+                                    http_multi_do, NULL, s);
+            break;
+        case CURL_POLL_REMOVE:
+            qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL);
+            break;
+    }
+
+    return 0;
+}
+#endif
+
+static size_t http_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
+{
+    HTTPState *s = ((HTTPState*)opaque);
+    size_t realsize = size * nmemb;
+
+    memcpy(s->buf, ptr, realsize);
+    s->buf += realsize;
+
+    printd("CURL: Just read %lld bytes\n", (unsigned long long)realsize);
+
+    return realsize;
+}
+
+static void http_multi_do(void *arg)
+{
+#ifdef CONFIG_AIO
+    BDRVHTTPState *s = (BDRVHTTPState *)arg;
+    int running;
+    int r;
+    int msgs_in_queue;
+
+    if (!s->multi)
+        return;
+
+    do {
+        r = curl_multi_socket_all(s->multi, &running);
+    } while(r == CURLM_CALL_MULTI_PERFORM);
+
+    /* Try to find done transfers, so we can free the easy
+     * handle again. */
+    do {
+        CURLMsg *msg;
+        msg = curl_multi_info_read(s->multi, &msgs_in_queue);
+
+        if (!msg)
+            break;
+        if (msg->msg == CURLMSG_NONE)
+            break;
+
+        switch (msg->msg) {
+            case CURLMSG_DONE:
+            {
+                HTTPState *state = NULL;
+                curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, 
(char**)&state);
+
+                // Finish processing this chunk
+                qemu_iovec_from_buffer(state->acb->qiov, state->orig_buf, 
state->buf_len);
+                qemu_free(state->orig_buf);
+                state->acb->common.cb(state->acb->common.opaque, 0);
+                qemu_aio_release(state->acb);
+                state->acb = NULL;
+
+                http_clean_state(state);
+                break;
+            }
+            default:
+                msgs_in_queue = 0;
+                break;
+        }
+    } while(msgs_in_queue);
+#endif
+}
+
+static HTTPState *http_init_state(BDRVHTTPState *s)
+{
+    HTTPState *state = NULL;
+    int i;
+
+    do {
+        for (i=0; i<HTTP_NUM_STATES; i++) {
+            if (!s->states[i].in_use) {
+                state = &s->states[i];
+                state->in_use = 1;
+                break;
+            }
+        }
+        if (!state) {
+            usleep(100);
+            http_multi_do(s);
+        }
+    } while(!state);
+
+    if (state->curl)
+        goto has_curl;
+
+    state->curl = curl_easy_init();
+    if (!state->curl)
+        return NULL;
+    curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
+    curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
+    curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, http_read_cb);
+    curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
+    curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
+    curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
+    curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
+    curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
+    curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
+    
+#ifdef DEBUG_VERBOSE
+    curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
+#endif
+
+has_curl:
+
+    state->s = s;
+
+    return state;
+}
+
+static void http_clean_state(HTTPState *s)
+{
+    if (s->s->multi)
+        curl_multi_remove_handle(s->s->multi, s->curl);
+    s->in_use = 0;
+}
+
+static int http_open(BlockDriverState *bs, const char *filename, int flags)
+{
+    BDRVHTTPState *s = bs->opaque;
+    HTTPState *state = NULL;
+    double d;
+    static int inited = 0;
+
+    if (!inited) {
+        curl_global_init(CURL_GLOBAL_ALL);
+        inited = 1;
+    }
+
+    printd("CURL: Opening %s\n", filename);
+    s->url = strdup(filename);
+    state = http_init_state(s);
+    if (!state)
+        goto out_noclean;
+
+    // Get file size
+
+    curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
+    if (curl_easy_perform(state->curl))
+        goto out;
+    curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
+    curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
+    if (!d)
+        goto out;
+    printd("CURL: Size = %lld\n", (long long)d);
+    s->len = (int64_t)d;
+
+    http_clean_state(state);
+    curl_easy_cleanup(state->curl);
+    state->curl = NULL;
+
+#ifdef CONFIG_AIO
+    // Now we know the file exists and its size, so let's
+    // initialize the multi interface!
+
+    s->multi = curl_multi_init();
+    curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s); 
+    curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, http_sock_cb ); 
+    http_multi_do(s);
+#endif /* CONFIG_AIO */
+
+    return 0;
+
+out:
+    fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
+    curl_easy_cleanup(state->curl);
+    state->curl = NULL;
+out_noclean:
+    return -EINVAL;
+}
+
+#ifdef CONFIG_AIO
+
+static BlockDriverAIOCB *http_aio_readv(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    BDRVHTTPState *s = bs->opaque;
+    HTTPAIOCB *acb;
+    long long start = sector_num * SECTOR_SIZE;
+    long long end = start + (nb_sectors * SECTOR_SIZE) - 1;
+    HTTPState *state;
+
+    acb = qemu_aio_get(bs, cb, opaque);
+    if (!acb)
+        return NULL;
+
+    state = http_init_state(s);
+    if (!state)
+        return NULL;
+
+    state->buf_len = (nb_sectors * SECTOR_SIZE);
+    state->buf = qemu_malloc(state->buf_len);
+    state->orig_buf = state->buf;
+    state->acb = acb;
+    acb->qiov = qiov;
+
+    snprintf(state->range, 127, "%lld-%lld", start, end);
+    printd("CURL (AIO): Reading %d at %lld (%s)\n", (nb_sectors * 
SECTOR_SIZE), start, state->range);
+    curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
+
+    curl_multi_add_handle(s->multi, state->curl);
+    http_multi_do(s);
+
+    return &acb->common;
+}
+
+#else
+
+static int http_read(BlockDriverState *bs, int64_t sector_num,
+                    uint8_t *buf, int nb_sectors)
+{
+    BDRVHTTPState *s = bs->opaque;
+    CURLcode res;
+    long long start = sector_num * SECTOR_SIZE;
+    long long end = start + (nb_sectors * SECTOR_SIZE) - 1;
+    HTTPState *state;
+
+    state = http_init_state(s);
+    if (!state)
+        return -1;
+
+    state->buf = buf;
+
+    // seek(sector_num * SECTOR_SIZE);
+    snprintf(state->range, 127, "%lld-%lld", start, end);
+    printd("CURL: Reading %d at %lld (%s)\n", (nb_sectors * SECTOR_SIZE), 
start, state->range);
+    curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
+
+    // read(nb_sectors * SECTOR_SIZE); -> buf
+    res = curl_easy_perform(state->curl);
+
+    http_clean_state(state);
+
+    printd("CURL: Done Reading\n");
+    return 0;
+}
+
+#endif /* CONFIG_AIO */
+
+static void http_close(BlockDriverState *bs)
+{
+    BDRVHTTPState *s = bs->opaque;
+    int i;
+
+    printd("CURL: Close\n");
+    for (i=0; i<HTTP_NUM_STATES; i++) {
+        if (s->states[i].in_use)
+            http_clean_state(&s->states[i]);
+        if (s->states[i].curl) {
+            curl_easy_cleanup(s->states[i].curl);
+            s->states[i].curl = NULL;
+        }
+    }
+    if (s->multi)
+        curl_multi_cleanup(s->multi);
+}
+
+static int64_t http_getlength(BlockDriverState *bs)
+{
+    BDRVHTTPState *s = bs->opaque;
+    return s->len;
+}
+
+BlockDriver bdrv_http = {
+    .format_name       = "http",
+    .protocol_name     = "http",
+    .instance_size     = sizeof(BDRVHTTPState),
+    .bdrv_open         = http_open,
+    .bdrv_close                = http_close,
+    .bdrv_getlength    = http_getlength,
+
+#ifdef CONFIG_AIO
+    .aiocb_size                = sizeof(HTTPAIOCB),
+    .bdrv_aio_readv    = http_aio_readv,
+#else
+    .bdrv_read         = http_read,
+#endif
+};
diff --git a/block.c b/block.c
index 3d1223d..76a8d86 100644
--- a/block.c
+++ b/block.c
@@ -1504,6 +1504,9 @@ void bdrv_init(void)
     bdrv_register(&bdrv_qcow2);
     bdrv_register(&bdrv_parallels);
     bdrv_register(&bdrv_nbd);
+#ifdef CONFIG_CURL
+    bdrv_register(&bdrv_http);
+#endif
 }
 
 void aio_pool_init(AIOPool *pool, int aiocb_size,
diff --git a/block.h b/block.h
index 5aef076..ebff06d 100644
--- a/block.h
+++ b/block.h
@@ -20,6 +20,7 @@ extern BlockDriver bdrv_vvfat;
 extern BlockDriver bdrv_qcow2;
 extern BlockDriver bdrv_parallels;
 extern BlockDriver bdrv_nbd;
+extern BlockDriver bdrv_http;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
diff --git a/configure b/configure
index 82fb60a..49b5b84 100755
--- a/configure
+++ b/configure
@@ -181,6 +181,7 @@ bsd_user="no"
 build_docs="no"
 uname_release=""
 curses="yes"
+curl="yes"
 pthread="yes"
 aio="yes"
 io_thread="no"
@@ -477,6 +478,8 @@ for opt do
   ;;
   --disable-curses) curses="no"
   ;;
+  --disable-curl) curl="no"
+  ;;
   --disable-nptl) nptl="no"
   ;;
   --enable-mixemu) mixemu="yes"
@@ -600,6 +603,7 @@ echo "  --disable-brlapi         disable BrlAPI"
 echo "  --disable-vnc-tls        disable TLS encryption for VNC server"
 echo "  --disable-vnc-sasl       disable SASL encryption for VNC server"
 echo "  --disable-curses         disable curses output"
+echo "  --disable-curl           disable curl connectivity"
 echo "  --disable-bluez          disable bluez stack connectivity"
 echo "  --disable-kvm            disable KVM acceleration support"
 echo "  --disable-nptl           disable usermode NPTL support"
@@ -1062,6 +1066,21 @@ EOF
 fi # test "$curses"
 
 ##########################################
+# curl probe
+
+if test "$curl" = "yes" ; then
+  curl=no
+  cat > $TMPC << EOF
+#include <curl/curl.h>
+int main(void) { return curl_easy_init(); }
+EOF
+  curl_libs=`curl-config --libs`
+ if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+    curl=yes
+  fi
+fi # test "$curl"
+
+##########################################
 # bluez support probe
 if test "$bluez" = "yes" ; then
   `pkg-config bluez 2> /dev/null` || bluez="no"
@@ -1311,6 +1330,7 @@ if test "$sdl" != "no" ; then
     echo "SDL static link   $sdl_static"
 fi
 echo "curses support    $curses"
+echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
 echo "Extra audio cards $audio_card_list"
@@ -1637,6 +1657,11 @@ fi
 if test "$inotify" = "yes" ; then
   echo "#define CONFIG_INOTIFY 1" >> $config_h
 fi
+if test "$curl" = "yes" ; then
+  echo "CONFIG_CURL=yes" >> $config_mak
+  echo "CURL_LIBS=$curl_libs" >> $config_mak
+  echo "#define CONFIG_CURL 1" >> $config_h
+fi
 if test "$brlapi" = "yes" ; then
   echo "CONFIG_BRLAPI=yes" >> $config_mak
   echo "#define CONFIG_BRLAPI 1" >> $config_h
-- 
1.6.0.2