[Qemu-devel] [RFC V3 18/24] qcow2: Integrate deduplication in qcow2_co_writev loop.

[Benoît Canet]

Signed-off-by: Benoit Canet <address@hidden>
---
 block/qcow2.c |   86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 1 deletion(-)

diff --git a/block/qcow2.c b/block/qcow2.c
index e641049..d5f28dd 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -330,6 +330,7 @@ static int qcow2_open(BlockDriverState *bs, int flags)
     QCowHeader header;
     uint64_t ext_end;
 
+    s->has_dedup = false;
     ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
     if (ret < 0) {
         goto fail;
@@ -812,11 +813,19 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState 
*bs,
     QEMUIOVector hd_qiov;
     uint64_t bytes_done = 0;
     uint8_t *cluster_data = NULL;
+    uint8_t *dedup_cluster_data = NULL;
+    uint8_t *next_call_first_hash;
+    int dedup_cluster_data_nr;
+    int deduped_sectors_nr;
+    int skip_before_dedup_clusters_nr;
+    int next_non_dedupable_sectors_nr;
+    UndedupableHashes u;
     QCowL2Meta l2meta = {
         .nb_clusters = 0,
         .oflag_copied = true,
         .overwrite = false,
     };
+    QTAILQ_INIT(&u.undedupable_hashes);
 
     trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
                                  remaining_sectors);
@@ -829,11 +838,67 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState 
*bs,
 
     qemu_co_mutex_lock(&s->lock);
 
+    if (s->has_dedup) {
+        /* if deduplication is on we make sure dedup_cluster_data
+         * contains a multiple of cluster size of data in order
+         * to compute the hashes
+         */
+        ret = qcow2_dedup_read_missing_and_concatenate(bs,
+                                                       qiov,
+                                                       sector_num,
+                                                       remaining_sectors,
+                                                       &dedup_cluster_data,
+                                                       &dedup_cluster_data_nr);
+
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    next_call_first_hash = NULL;
+    next_non_dedupable_sectors_nr = 0;
+    skip_before_dedup_clusters_nr = 0;
     while (remaining_sectors != 0) {
 
         trace_qcow2_writev_start_part(qemu_coroutine_self());
+
+        if (s->has_dedup && next_non_dedupable_sectors_nr == 0) {
+            /* Try to deduplicate as much clusters as possible */
+            deduped_sectors_nr = qcow2_dedup(bs,
+                                             &u,
+                                             sector_num,
+                                             dedup_cluster_data,
+                                             dedup_cluster_data_nr,
+                                             &skip_before_dedup_clusters_nr,
+                                             &next_non_dedupable_sectors_nr,
+                                             &next_call_first_hash);
+
+            remaining_sectors -= deduped_sectors_nr;
+            sector_num += deduped_sectors_nr;
+            bytes_done += deduped_sectors_nr * 512;
+
+            /* no more data to write -> exit
+             * Can be < 0 because of the presence of sectors we read in
+             * qcow2_read_missing_dedup_sectors_and_concatenate.
+             */
+            if (next_non_dedupable_sectors_nr <= 0) {
+                goto fail;
+            }
+
+            /* if we deduped something trace it */
+            if (deduped_sectors_nr) {
+                trace_qcow2_writev_done_part(qemu_coroutine_self(),
+                                             deduped_sectors_nr);
+                trace_qcow2_writev_start_part(qemu_coroutine_self());
+            }
+        }
+
         index_in_cluster = sector_num & (s->cluster_sectors - 1);
-        n_end = index_in_cluster + remaining_sectors;
+        n_end = s->has_dedup &&
+                next_non_dedupable_sectors_nr < remaining_sectors ?
+                index_in_cluster + next_non_dedupable_sectors_nr :
+                index_in_cluster + remaining_sectors;
+
         if (s->crypt_method &&
             n_end > QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors) {
             n_end = QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors;
@@ -875,6 +940,23 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState 
*bs,
                 cur_nr_sectors * 512);
         }
 
+        /* Write the non duplicated clusters hashes to disk */
+        if (s->has_dedup) {
+            int count = cur_nr_sectors / s->cluster_sectors;
+            int has_ending = ((cluster_offset >> 9) + index_in_cluster +
+                             cur_nr_sectors) & (s->cluster_sectors - 1);
+            count = index_in_cluster ? count + 1 : count;
+            count = has_ending ? count + 1 : count;
+            ret = qcow2_dedup_write_new_hashes(bs,
+                                               &u,
+                                               count,
+                                               sector_num,
+                                               (cluster_offset >> 9));
+            if (ret < 0) {
+                goto fail;
+            }
+        }
+
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         qemu_co_mutex_unlock(&s->lock);
         trace_qcow2_writev_data(qemu_coroutine_self(),
@@ -894,6 +976,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState 
*bs,
 
         run_dependent_requests(s, &l2meta);
 
+        next_non_dedupable_sectors_nr -= cur_nr_sectors;
         remaining_sectors -= cur_nr_sectors;
         sector_num += cur_nr_sectors;
         bytes_done += cur_nr_sectors * 512;
@@ -908,6 +991,7 @@ fail:
 
     qemu_iovec_destroy(&hd_qiov);
     qemu_vfree(cluster_data);
+    qemu_vfree(dedup_cluster_data);
     trace_qcow2_writev_done_req(qemu_coroutine_self(), ret);
 
     return ret;
-- 
1.7.10.4