[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Qemu-devel] [PATCH] Add HTTP protocol using curl v3
|
From: |
alex |
|
Subject: |
[Qemu-devel] [PATCH] Add HTTP protocol using curl v3 |
|
Date: |
Thu, 7 May 2009 13:11:00 +0200 |
From: Alexander Graf <address@hidden>
Currently Qemu can read from posix I/O and NBD. This patch adds a
third protocol to the game: HTTP.
In certain situations it can be useful to access HTTP data directly,
for example if you want to try out an http provided OS image, but
don't know if you want to download it yet.
Using this patch you can now try it on on the fly. Just use it like:
qemu -cdrom http://host/path/my.iso
In order to not reinvent the wheel, this patch uses libcurl.
v2 changes:
- fix the segfault (yay!)
- implement AIO
v3 changes:
- remove synchronous API
- implement caching
Signed-off-by: Alexander Graf <address@hidden>
---
Makefile | 6 +
Makefile.target | 2 +-
block-http.c | 431 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
block.c | 3 +
block.h | 1 +
configure | 25 ++++
6 files changed, 467 insertions(+), 1 deletions(-)
create mode 100644 block-http.c
diff --git a/Makefile b/Makefile
index 3c70068..fbfbc6e 100644
--- a/Makefile
+++ b/Makefile
@@ -77,6 +77,10 @@ endif
BLOCK_OBJS += block-raw-posix.o
endif
+ifdef CONFIG_CURL
+BLOCK_OBJS += block-http.o
+endif
+
######################################################################
# libqemu_common.a: Target independent part of system emulation. The
# long term path is to suppress *all* target specific code in case of
@@ -185,6 +189,8 @@ endif
LIBS+=$(VDE_LIBS)
+LIBS+=$(CURL_LIBS)
+
cocoa.o: cocoa.m
keymaps.o: keymaps.c keymaps.h
diff --git a/Makefile.target b/Makefile.target
index f735105..3a9226c 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -735,7 +735,7 @@ endif
vl.o: qemu-options.h
-$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS)
$(VDE_LIBS)
+$(QEMU_PROG): LIBS += $(SDL_LIBS) $(COCOA_LIBS) $(CURSES_LIBS) $(BRLAPI_LIBS)
$(VDE_LIBS) $(CURL_LIBS)
$(QEMU_PROG): $(OBJS) ../libqemu_common.a libqemu.a
$(LINK)
diff --git a/block-http.c b/block-http.c
new file mode 100644
index 0000000..6825cba
--- /dev/null
+++ b/block-http.c
@@ -0,0 +1,431 @@
+/*
+ * QEMU Block driver for HTTP images
+ *
+ * Copyright (c) 2009 Alexander Graf <address@hidden>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "block_int.h"
+#include <curl/curl.h>
+
+// #define DEBUG
+// #define DEBUG_VERBOSE
+
+#ifdef DEBUG
+#define printd printf
+#else
+#define printd(x, ...)
+#endif
+
+#define HTTP_NUM_STATES 8
+#define HTTP_NUM_ACB 8
+#define SECTOR_SIZE 512
+#define READ_AHEAD_SIZE (256 * 1024)
+
+#define FIND_RET_NONE 0
+#define FIND_RET_OK 1
+#define FIND_RET_WAIT 2
+
+struct BDRVHTTPState;
+
+typedef struct HTTPAIOCB {
+ BlockDriverAIOCB common;
+ QEMUIOVector *qiov;
+ int64_t start;
+ int64_t end;
+} HTTPAIOCB;
+
+typedef struct HTTPState
+{
+ struct BDRVHTTPState *s;
+ HTTPAIOCB *acb[HTTP_NUM_ACB];
+ CURL *curl;
+ char *orig_buf;
+ int64_t buf_start;
+ int64_t buf_off;
+ int64_t buf_len;
+ char range[128];
+ char errmsg[CURL_ERROR_SIZE];
+ char in_use;
+} HTTPState;
+
+typedef struct BDRVHTTPState {
+ CURLM *multi;
+ int64_t len;
+ HTTPState states[HTTP_NUM_STATES];
+ char *url;
+} BDRVHTTPState;
+
+static void http_clean_state(HTTPState *s);
+static void http_multi_do(void *arg);
+
+static int http_sock_cb(CURL *curl, curl_socket_t fd, int action,
+ void *s, void *sp)
+{
+ printd("CURL (AIO): Sock action %d on fd %d\n", action, fd);
+ switch (action) {
+ case CURL_POLL_IN:
+ qemu_aio_set_fd_handler(fd, http_multi_do, NULL, NULL, s);
+ break;
+ case CURL_POLL_OUT:
+ qemu_aio_set_fd_handler(fd, NULL, http_multi_do, NULL, s);
+ break;
+ case CURL_POLL_INOUT:
+ qemu_aio_set_fd_handler(fd, http_multi_do,
+ http_multi_do, NULL, s);
+ break;
+ case CURL_POLL_REMOVE:
+ qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL);
+ break;
+ }
+
+ return 0;
+}
+
+static size_t http_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
+{
+ HTTPState *s = ((HTTPState*)opaque);
+ size_t realsize = size * nmemb;
+ int i;
+
+ memcpy(s->orig_buf + s->buf_off, ptr, realsize);
+ s->buf_off += realsize;
+
+ for(i=0; i<HTTP_NUM_ACB; i++) {
+ HTTPAIOCB *acb = s->acb[i];
+
+ if (!acb)
+ continue;
+
+ if ((s->buf_off >= acb->end)) {
+ qemu_iovec_from_buffer(acb->qiov, s->orig_buf + acb->start,
+ acb->end - acb->start);
+ acb->common.cb(acb->common.opaque, 0);
+ qemu_aio_release(acb);
+ s->acb[i] = NULL;
+ }
+ }
+
+ printd("CURL: Just read %lld bytes\n", (unsigned long long)realsize);
+
+ return realsize;
+}
+
+static int http_find_buf(BDRVHTTPState *s, long long start, long long len,
+ HTTPAIOCB *acb)
+{
+ int i;
+ long long end = start + len;
+
+ for (i=0; i<HTTP_NUM_STATES; i++) {
+ HTTPState *state = &s->states[i];
+ long long buf_end = (state->buf_start + state->buf_off);
+ long long buf_fend = (state->buf_start + state->buf_len);
+
+ if (!state->orig_buf)
+ continue;
+ if (!state->buf_off)
+ continue;
+
+ // Does the existing buffer cover our section?
+ if ((start >= state->buf_start) &&
+ (start <= buf_end) &&
+ (end >= state->buf_start) &&
+ (end <= buf_end))
+ {
+ char *buf = state->orig_buf + (start - state->buf_start);
+
+ qemu_iovec_from_buffer(acb->qiov, buf, len);
+ acb->common.cb(acb->common.opaque, 0);
+
+ return FIND_RET_OK;
+ }
+
+ // Wait for unfinished chunks
+ if ((start >= state->buf_start) &&
+ (start <= buf_fend) &&
+ (end >= state->buf_start) &&
+ (end <= buf_fend))
+ {
+ int j;
+
+ acb->start = start - state->buf_start;
+ acb->end = acb->start + len;
+
+ for (j=0; j<HTTP_NUM_ACB; j++) {
+ if (!state->acb[j]) {
+ state->acb[j] = acb;
+ return FIND_RET_WAIT;
+ }
+ }
+ }
+ }
+
+ return FIND_RET_NONE;
+}
+
+static void http_multi_do(void *arg)
+{
+ BDRVHTTPState *s = (BDRVHTTPState *)arg;
+ int running;
+ int r;
+ int msgs_in_queue;
+
+ if (!s->multi)
+ return;
+
+ do {
+ r = curl_multi_socket_all(s->multi, &running);
+ } while(r == CURLM_CALL_MULTI_PERFORM);
+
+ /* Try to find done transfers, so we can free the easy
+ * handle again. */
+ do {
+ CURLMsg *msg;
+ msg = curl_multi_info_read(s->multi, &msgs_in_queue);
+
+ if (!msg)
+ break;
+ if (msg->msg == CURLMSG_NONE)
+ break;
+
+ switch (msg->msg) {
+ case CURLMSG_DONE:
+ {
+ HTTPState *state = NULL;
+ curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE,
(char**)&state);
+ http_clean_state(state);
+ break;
+ }
+ default:
+ msgs_in_queue = 0;
+ break;
+ }
+ } while(msgs_in_queue);
+}
+
+static HTTPState *http_init_state(BDRVHTTPState *s)
+{
+ HTTPState *state = NULL;
+ int i, j;
+
+ do {
+ for (i=0; i<HTTP_NUM_STATES; i++) {
+ for (j=0; j<HTTP_NUM_ACB; j++)
+ if (s->states[i].acb[j])
+ continue;
+ if (s->states[i].in_use)
+ continue;
+
+ state = &s->states[i];
+ state->in_use = 1;
+ break;
+ }
+ if (!state) {
+ usleep(100);
+ http_multi_do(s);
+ }
+ } while(!state);
+
+ if (state->curl)
+ goto has_curl;
+
+ state->curl = curl_easy_init();
+ if (!state->curl)
+ return NULL;
+ curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
+ curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
+ curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, http_read_cb);
+ curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
+ curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
+ curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
+ curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
+ curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
+ curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
+
+#ifdef DEBUG_VERBOSE
+ curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
+#endif
+
+has_curl:
+
+ state->s = s;
+
+ return state;
+}
+
+static void http_clean_state(HTTPState *s)
+{
+ if (s->s->multi)
+ curl_multi_remove_handle(s->s->multi, s->curl);
+ s->in_use = 0;
+}
+
+static int http_open(BlockDriverState *bs, const char *filename, int flags)
+{
+ BDRVHTTPState *s = bs->opaque;
+ HTTPState *state = NULL;
+ double d;
+ static int inited = 0;
+
+ if (!inited) {
+ curl_global_init(CURL_GLOBAL_ALL);
+ inited = 1;
+ }
+
+ printd("CURL: Opening %s\n", filename);
+ s->url = strdup(filename);
+ state = http_init_state(s);
+ if (!state)
+ goto out_noclean;
+
+ // Get file size
+
+ curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
+ if (curl_easy_perform(state->curl))
+ goto out;
+ curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
+ curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
+ if (!d)
+ goto out;
+ printd("CURL: Size = %lld\n", (long long)d);
+ s->len = (int64_t)d;
+
+ http_clean_state(state);
+ curl_easy_cleanup(state->curl);
+ state->curl = NULL;
+
+ // Now we know the file exists and its size, so let's
+ // initialize the multi interface!
+
+ s->multi = curl_multi_init();
+ curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
+ curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, http_sock_cb );
+ http_multi_do(s);
+
+ return 0;
+
+out:
+ fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
+ curl_easy_cleanup(state->curl);
+ state->curl = NULL;
+out_noclean:
+ return -EINVAL;
+}
+
+static BlockDriverAIOCB *http_aio_readv(BlockDriverState *bs,
+ int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+ BlockDriverCompletionFunc *cb, void *opaque)
+{
+ BDRVHTTPState *s = bs->opaque;
+ HTTPAIOCB *acb;
+ long long start = sector_num * SECTOR_SIZE;
+ long long end;
+ HTTPState *state;
+
+ acb = qemu_aio_get(bs, cb, opaque);
+ if (!acb)
+ return NULL;
+
+ acb->qiov = qiov;
+
+ // In case we have the requested data already (e.g. read-ahead),
+ // we can just call the callback and be done.
+
+ switch (http_find_buf(s, start, nb_sectors * SECTOR_SIZE, acb)) {
+ case FIND_RET_OK:
+ qemu_aio_release(acb);
+ // fall through
+ case FIND_RET_WAIT:
+ return &acb->common;
+ default:
+ break;
+ }
+
+ // No cache found, so let's start a new request
+
+ state = http_init_state(s);
+ if (!state)
+ return NULL;
+
+ acb->start = 0;
+ acb->end = (nb_sectors * SECTOR_SIZE);
+
+ state->buf_off = 0;
+ if (state->orig_buf)
+ qemu_free(state->orig_buf);
+ state->buf_start = start;
+ state->buf_len = acb->end + READ_AHEAD_SIZE;
+ end = MIN(start + state->buf_len, s->len) - 1;
+ state->orig_buf = qemu_malloc(state->buf_len);
+ state->acb[0] = acb;
+
+ snprintf(state->range, 127, "%lld-%lld", start, end);
+ printd("CURL (AIO): Reading %d at %lld (%s)\n", (nb_sectors *
SECTOR_SIZE), start, state->range);
+ curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
+
+ curl_multi_add_handle(s->multi, state->curl);
+ http_multi_do(s);
+
+ return &acb->common;
+}
+
+static void http_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+ // Do we have to implement canceling? Seems to work without...
+}
+
+static void http_close(BlockDriverState *bs)
+{
+ BDRVHTTPState *s = bs->opaque;
+ int i;
+
+ printd("CURL: Close\n");
+ for (i=0; i<HTTP_NUM_STATES; i++) {
+ if (s->states[i].in_use)
+ http_clean_state(&s->states[i]);
+ if (s->states[i].curl) {
+ curl_easy_cleanup(s->states[i].curl);
+ s->states[i].curl = NULL;
+ }
+ }
+ if (s->multi)
+ curl_multi_cleanup(s->multi);
+}
+
+static int64_t http_getlength(BlockDriverState *bs)
+{
+ BDRVHTTPState *s = bs->opaque;
+ return s->len;
+}
+
+BlockDriver bdrv_http = {
+ .format_name = "http",
+ .protocol_name = "http",
+
+ .instance_size = sizeof(BDRVHTTPState),
+ .bdrv_open = http_open,
+ .bdrv_close = http_close,
+ .bdrv_getlength = http_getlength,
+
+ .aiocb_size = sizeof(HTTPAIOCB),
+ .bdrv_aio_readv = http_aio_readv,
+ .bdrv_aio_cancel = http_aio_cancel,
+};
diff --git a/block.c b/block.c
index 3d1223d..76a8d86 100644
--- a/block.c
+++ b/block.c
@@ -1504,6 +1504,9 @@ void bdrv_init(void)
bdrv_register(&bdrv_qcow2);
bdrv_register(&bdrv_parallels);
bdrv_register(&bdrv_nbd);
+#ifdef CONFIG_CURL
+ bdrv_register(&bdrv_http);
+#endif
}
void aio_pool_init(AIOPool *pool, int aiocb_size,
diff --git a/block.h b/block.h
index 5aef076..ebff06d 100644
--- a/block.h
+++ b/block.h
@@ -20,6 +20,7 @@ extern BlockDriver bdrv_vvfat;
extern BlockDriver bdrv_qcow2;
extern BlockDriver bdrv_parallels;
extern BlockDriver bdrv_nbd;
+extern BlockDriver bdrv_http;
typedef struct BlockDriverInfo {
/* in bytes, 0 if irrelevant */
diff --git a/configure b/configure
index 82fb60a..49b5b84 100755
--- a/configure
+++ b/configure
@@ -181,6 +181,7 @@ bsd_user="no"
build_docs="no"
uname_release=""
curses="yes"
+curl="yes"
pthread="yes"
aio="yes"
io_thread="no"
@@ -477,6 +478,8 @@ for opt do
;;
--disable-curses) curses="no"
;;
+ --disable-curl) curl="no"
+ ;;
--disable-nptl) nptl="no"
;;
--enable-mixemu) mixemu="yes"
@@ -600,6 +603,7 @@ echo " --disable-brlapi disable BrlAPI"
echo " --disable-vnc-tls disable TLS encryption for VNC server"
echo " --disable-vnc-sasl disable SASL encryption for VNC server"
echo " --disable-curses disable curses output"
+echo " --disable-curl disable curl connectivity"
echo " --disable-bluez disable bluez stack connectivity"
echo " --disable-kvm disable KVM acceleration support"
echo " --disable-nptl disable usermode NPTL support"
@@ -1062,6 +1066,21 @@ EOF
fi # test "$curses"
##########################################
+# curl probe
+
+if test "$curl" = "yes" ; then
+ curl=no
+ cat > $TMPC << EOF
+#include <curl/curl.h>
+int main(void) { return curl_easy_init(); }
+EOF
+ curl_libs=`curl-config --libs`
+ if $cc $ARCH_CFLAGS $curl_libs -o $TMPE $TMPC > /dev/null 2> /dev/null ; then
+ curl=yes
+ fi
+fi # test "$curl"
+
+##########################################
# bluez support probe
if test "$bluez" = "yes" ; then
`pkg-config bluez 2> /dev/null` || bluez="no"
@@ -1311,6 +1330,7 @@ if test "$sdl" != "no" ; then
echo "SDL static link $sdl_static"
fi
echo "curses support $curses"
+echo "curl support $curl"
echo "mingw32 support $mingw32"
echo "Audio drivers $audio_drv_list"
echo "Extra audio cards $audio_card_list"
@@ -1637,6 +1657,11 @@ fi
if test "$inotify" = "yes" ; then
echo "#define CONFIG_INOTIFY 1" >> $config_h
fi
+if test "$curl" = "yes" ; then
+ echo "CONFIG_CURL=yes" >> $config_mak
+ echo "CURL_LIBS=$curl_libs" >> $config_mak
+ echo "#define CONFIG_CURL 1" >> $config_h
+fi
if test "$brlapi" = "yes" ; then
echo "CONFIG_BRLAPI=yes" >> $config_mak
echo "#define CONFIG_BRLAPI 1" >> $config_h
--
1.6.0.2
- [Qemu-devel] [PATCH] Add HTTP protocol using curl v3,
alex <=