[Qemu-devel] [Patch 3/4] [RFC] Zero Cluster Dedup, Offline dedup, qemu-img extentions

[Shahar Frank]

MDSUM Extension for the map verb

Signed-off-by: Shahar Frank <address@hidden>

diff --git a/Makefile b/Makefile
index 35061a4..21e43d5 100644
--- a/Makefile
+++ b/Makefile
@@ -187,7 +187,7 @@ libqemu_user.a: $(USER_OBJS)
 ######################################################################
 
 qemu-img$(EXESUF): qemu-img.o qemu-tool.o osdep.o $(BLOCK_OBJS)
-       $(CC) $(LDFLAGS) -o $@ $^ -lz $(LIBS)
+       $(CC) $(LDFLAGS) -o $@ $^ -lz $(LIBS) $(MD5SUMLIBS) 
 
 %.o: %.c
        $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
diff --git a/configure b/configure
index a224e6a..6420066 100755
--- a/configure
+++ b/configure
@@ -114,6 +114,7 @@ mixemu="no"
 bluez="yes"
 signalfd="no"
 eventfd="no"
+md5sum="yes"
 
 # OS specific
 targetos=`uname -s`
@@ -346,6 +347,8 @@ for opt do
   ;;
   --disable-aio) aio="no"
   ;;
+  --disable-md5sum) md5sum="no"
+  ;;
   *) echo "ERROR: unknown option $opt"; show_help="yes"
   ;;
   esac
@@ -448,6 +451,7 @@ echo "  --enable-uname-release=R Return R for uname
-r in usermode emulation"
 echo "  --sparc_cpu=V            Build qemu for Sparc architecture v7,
v8, v8plus, v8plusa, v9"
 echo "  --disable-vde            disable support for vde network"
 echo "  --disable-aio            disable AIO support"
+echo "  --disable-md5sum         disable MD5SUM support"
 echo ""
 echo "NOTE: The object files are built at the place where configure is
launched"
 exit 1
@@ -489,6 +493,7 @@ if test "$mingw32" = "yes" ; then
     linux_user="no"
 fi
 
+MD5SUMLIBS=
 if [ "$darwin" = "yes" -o "$mingw32" = "yes" ] ; then
     AIOLIBS=
 elif [ "$bsd" = "yes" ]; then
@@ -496,6 +501,7 @@ elif [ "$bsd" = "yes" ]; then
 else
     # Some Linux architectures (e.g. s390) don't imply -lpthread
automatically.
     AIOLIBS="-lrt -lpthread"
+    MD5SUMLIBS="-lssl"
 fi
 
 # Check for gcc4, error if pre-gcc4
@@ -920,6 +926,20 @@ EOF
 fi
 
 ##########################################
+# md5sum probe
+if test "$md5sum" = "yes" ; then
+  md5sum=no
+  cat > $TMPC << EOF
+#include <openssl/bio.h>
+#include <openssl/evp.h>
+int main(void) { return (int)BIO_new(BIO_s_mem()); }
+EOF
+  if $cc $ARCH_CFLAGS -o $TMPE $MD5SUMLIBS $TMPC 2> /dev/null ; then
+    md5sum=yes
+  fi
+fi
+
+##########################################
 # signalfd probe
 cat > $TMPC << EOF
 #define _GNU_SOURCE
@@ -1018,6 +1038,7 @@ echo "uname -r          $uname_release"
 echo "NPTL support      $nptl"
 echo "vde support       $vde"
 echo "AIO support       $aio"
+echo "md5sum support    $md5sum"
 
 if test $sdl_too_old = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL
support"
@@ -1064,6 +1085,7 @@ echo "CFLAGS=$CFLAGS" >> $config_mak
 echo "LDFLAGS=$LDFLAGS" >> $config_mak
 echo "EXESUF=$EXESUF" >> $config_mak
 echo "AIOLIBS=$AIOLIBS" >> $config_mak
+echo "MD5SUMLIBS=$MD5SUMLIBS" >> $config_mak
 case "$cpu" in
   i386)
     echo "ARCH=i386" >> $config_mak
@@ -1280,6 +1302,10 @@ if test "$aio" = "yes" ; then
   echo "#define CONFIG_AIO 1" >> $config_h
   echo "CONFIG_AIO=yes" >> $config_mak
 fi
+if test "$md5sum" = "yes" ; then
+  echo "#define CONFIG_MD5SUM 1" >> $config_h
+  echo "CONFIG_MD5SUM=yes" >> $config_mak
+fi
 if test "$signalfd" = "yes" ; then
   echo "#define CONFIG_signalfd 1" >> $config_h
 fi
diff --git a/qemu-img.c b/qemu-img.c
index c6fe43d..8cda447 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -77,6 +77,7 @@ static void help(void)
            "  '-e' indicates that the target image must be encrypted
(qcow format only)\n"
            "  '-6' indicates that the target image must use
compatibility level 6 (vmdk format only)\n"
            "  '-r' indicated that the base images are recursed in (for
info and map)\n"
+           "  '-s' indicated that the md5sum of the clusters is to be
printed (for map)\n"
           );
     printf("\nSupported format:");
     bdrv_iterate_format(format_print, NULL);
@@ -172,6 +173,79 @@ static int read_password(char *buf, int buf_size)
 }
 #endif
 
+
+#ifdef CONFIG_MD5SUM
+/* md5sum functiuons */
+
+#include <openssl/bio.h>
+#include <openssl/evp.h>
+
+#define MD5HASHSZ              16
+
+/* openssl buffering */
+BIO *bio;
+BIO *bio_md;
+
+static inline int update_hash_block(void *block, int len)
+{
+    return BIO_write(bio, block, len);
+}
+
+static inline int reset_hashing(void)
+{
+    return BIO_reset(bio);
+}
+
+static inline int get_hash(char *buf, int len)
+{
+    return BIO_gets(bio_md, buf, len);
+}
+
+static int compute_block_hash(void *buf, int bufsz, void *mdbuf, int
mdlen)
+{
+    int hashlen;
+    
+    reset_hashing();
+    update_hash_block(buf, bufsz);
+    if ((hashlen = get_hash(mdbuf, mdlen)) < 0)
+        return -1;
+    return hashlen;
+}
+
+static int sprint_hash(char *buf, int bufsz, uint8_t * hash, int
hashsz)
+{
+    int i;
+    
+    if (bufsz < hashsz * 2)
+        hashsz = bufsz / 2;
+    for (i = 0; i < hashsz; i++)
+        sprintf(buf + i * 2, "%02x", hash[i]);
+    if (i * 2 < bufsz)
+        buf[i * 2 + 1] = 0;
+    return i*2;
+}
+
+static int initialize_hashing(void)
+{
+    bio = BIO_new(BIO_s_mem());
+    if (!bio)
+        error("can't allocate bio struct");
+    bio_md = BIO_new(BIO_f_md());
+    if (!bio_md)
+        error("can't allocate bio_md struct");
+    if (BIO_set_md(bio_md, EVP_md5()) != 1)
+        error("can't set bio md");
+    bio = BIO_push(bio_md, bio);
+    if (!bio)
+        error("can't push hashing md to hashing bio");
+    reset_hashing();
+    return 0;
+}
+
+#else
+static int initialize_hashing(void) {};
+#endif
+
 static BlockDriverState *bdrv_new_open(const char *filename,
                                        const char *fmt)
 {
@@ -348,6 +422,33 @@ static int img_commit(int argc, char **argv)
 
 #define IO_BUF_SIZE 65536
 
+#ifdef CONFIG_MD5SUM
+static void
+sectors_sum_str(BlockDriverState *bs, uint64_t offset, int sectors,
char *str, int len)
+{
+    char buf[IO_BUF_SIZE];
+    char mdbuf[MD5HASHSZ];
+    
+    if (sectors <= 0 || sectors > (IO_BUF_SIZE >> 9))
+        error("sectors_sum_str: bad sectors param %d", sectors);
+    if (bdrv_read(bs, offset >> 9, buf, sectors) < 0)
+        error("sectors_sum_str: error while reading offset 0x%"PRIx64,
offset);
+    
+    if (compute_block_hash(buf, sectors << 9, mdbuf, sizeof mdbuf) < 0)
+        error("sectors_sum_str: can't compute hash");
+    
+    sprint_hash(str, len, mdbuf, sizeof mdbuf);
+}
+#else
+static void
+sectors_sum_str(BlockDriverState *bs, uint64_t offset,  int sectors,
char *str, int len)
+{
+    if (len > 0)
+        str[0] = 0;
+    return;
+}
+#endif
+
 static int img_map(int argc, char **argv)
 {
     int c, ret = 0, cluster_size, cluster_sectors;
@@ -357,11 +458,12 @@ static int img_map(int argc, char **argv)
     int64_t offset, poffset, e;
     uint64_t bs_sectors;
     BlockDriverInfo bdi;
-    int local = 1;
+    int local = 1, sum = 0;
+    char sumstr[64];
     
     fmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:hr");
+        c = getopt(argc, argv, "f:hrs");
         if (c == -1)
             break;
         switch(c) {
@@ -374,6 +476,15 @@ static int img_map(int argc, char **argv)
         case 'r':
             local = 0;
             break;
+        case 's':
+#ifdef CONFIG_MD5SUM
+            sum = 1;
+            sumstr[0] = ' ';
+            sumstr[1] = 0;
+#else
+            fprintf(stderr, "warning: md5sum is not supported\n");
+#endif
+            break;
         }
     }
     if (optind >= argc)
@@ -408,12 +519,14 @@ static int img_map(int argc, char **argv)
     for (offset = 0; offset < e; offset += cluster_size) {
         poffset = bs->drv->bdrv_map(bs, offset, local, &store);
         if (poffset > 0 || poffset == -ENOTSUP) {
+            if (sum)
+                sectors_sum_str(bs, offset, cluster_sectors, sumstr+1,
sizeof sumstr - 1);
             if (poffset == -ENOTSUP)
                 poffset = 0;
-            printf("%s 0x%016"PRIx64" 0x%016"PRIx64"\n",
-                   store->filename, offset, poffset);
+            printf("%s 0x%016"PRIx64" 0x%016"PRIx64"%s\n",
+                   store->filename, offset, poffset, sumstr);
         }
-       if (poffset < 0) {
+        if (poffset < 0) {
             ret = poffset;
             break;
         }
@@ -897,6 +1010,7 @@ int main(int argc, char **argv)
     const char *cmd;
     int ret = 0;
 
+    initialize_hashing();
     bdrv_init();
     if (argc < 2)
         help();

3-md5sum.patch
Description: 3-md5sum.patch