Re: [Qemu-devel] [PATCH v4 5/8] XBZRLE: optimize XBZRLE to decrease the cache misses

[陈梁]

> I've got a world with just patches 1..5 on that's seeing corruptions, but
> I've not seen where the problem is.  So far the world with 1..4 on hasn't
> hit those corruption, but maybe I need to test more.
> 
> Have you tested this set with google stressapptest?
> 
> Let it migrate for a few cycles with stress apptest running, then ctrl-z
> the stressapptest program to let the migration complete, then fg it
> to collect the results.
> 
> Dave
> 
> * address@hidden (address@hidden) wrote:
>> From: ChenLiang <address@hidden>
>> 
>> Avoid hot pages being replaced by others to remarkably decrease cache
>> misses
>> 
>> Sample results with the test program which quote from xbzrle.txt ran in
>> vm:(migrate bandwidth:1GE and xbzrle cache size 8MB)
>> 
>> the test program:
>> 
>> include <stdlib.h>
>> include <stdio.h>
>> int main()
>> {
>>        char *buf = (char *) calloc(4096, 4096);
>>        while (1) {
>>            int i;
>>            for (i = 0; i < 4096 * 4; i++) {
>>                buf[i * 4096 / 4]++;
>>            }
>>            printf(".");
>>        }
>> }
>> 
>> before this patch:
>> virsh qemu-monitor-command test_vm '{"execute": "query-migrate"}'
>> {"return":{"expected-downtime":1020,"xbzrle-cache":{"bytes":1108284,
>> "cache-size":8388608,"cache-miss-rate":0.987013,"pages":18297,"overflow":8,
>> "cache-miss":1228737},"status":"active","setup-time":10,"total-time":52398,
>> "ram":{"total":12466991104,"remaining":1695744,"mbps":935.559472,
>> "transferred":5780760580,"dirty-sync-counter":271,"duplicate":2878530,
>> "dirty-pages-rate":29130,"skipped":0,"normal-bytes":5748592640,
>> "normal":1403465}},"id":"libvirt-706"}
>> 
>> 18k pages sent compressed
>> cache-miss-rate is 98.7%, totally miss.
>> 
>> after optimizing:
>> virsh qemu-monitor-command test_vm '{"execute": "query-migrate"}'
>> {"return":{"expected-downtime":2054,"xbzrle-cache":{"bytes":5066763,
>> "cache-size":8388608,"cache-miss-rate":0.485924,"pages":194823,"overflow":0,
>> "cache-miss":210653},"status":"active","setup-time":11,"total-time":18729,
>> "ram":{"total":12466991104,"remaining":3895296,"mbps":937.663549,
>> "transferred":1615042219,"dirty-sync-counter":98,"duplicate":2869840,
>> "dirty-pages-rate":58781,"skipped":0,"normal-bytes":1588404224,
>> "normal":387794}},"id":"libvirt-266"}
>> 
>> 194k pages sent compressed
>> The value of cache-miss-rate decrease to 48.59%.
>> 
>> Signed-off-by: ChenLiang <address@hidden>
>> Signed-off-by: Gonglei <address@hidden>
>> Reviewed-by: Eric Blake <address@hidden>
>> ---
>> arch_init.c                    |  8 +++++---
>> docs/xbzrle.txt                |  8 ++++++++
>> include/migration/page_cache.h | 10 +++++++---
>> page_cache.c                   | 23 +++++++++++++++++++----
>> 4 files changed, 39 insertions(+), 10 deletions(-)
>> 
>> diff --git a/arch_init.c b/arch_init.c
>> index 15ca4c0..84a4bd3 100644
>> --- a/arch_init.c
>> +++ b/arch_init.c
>> @@ -343,7 +343,8 @@ static void xbzrle_cache_zero_page(ram_addr_t 
>> current_addr)
>> 
>>     /* We don't care if this fails to allocate a new cache page
>>      * as long as it updated an old one */
>> -    cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE);
>> +    cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE,
>> +                 bitmap_sync_count);
>> }
>> 
>> #define ENCODING_FLAG_XBZRLE 0x1
>> @@ -355,10 +356,11 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t 
>> **current_data,
>>     int encoded_len = 0, bytes_sent = -1;
>>     uint8_t *prev_cached_page;
>> 
>> -    if (!cache_is_cached(XBZRLE.cache, current_addr)) {
>> +    if (!cache_is_cached(XBZRLE.cache, current_addr, bitmap_sync_count)) {
>>         acct_info.xbzrle_cache_miss++;
>>         if (!last_stage) {
>> -            if (cache_insert(XBZRLE.cache, current_addr, *current_data) == 
>> -1) {
>> +            if (cache_insert(XBZRLE.cache, current_addr, *current_data,
>> +                             bitmap_sync_count) == -1) {
>>                 return -1;
>>             } else {
>>                 /* update *current_data when the page has been
>> diff --git a/docs/xbzrle.txt b/docs/xbzrle.txt
>> index cc3a26a..52c8511 100644
>> --- a/docs/xbzrle.txt
>> +++ b/docs/xbzrle.txt
>> @@ -71,6 +71,14 @@ encoded buffer:
>> encoded length 24
>> e9 07 0f 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 03 01 67 01 01 69
>> 
>> +Cache update strategy
>> +=====================
>> +Keeping the hot pages in the cache is effective for decreased cache
>> +misses. XBZRLE uses a counter as the age of each page. The counter will
>> +increase after each ram dirty bitmap sync. When a cache conflict is
>> +detected, XBZRLE will only evict pages in the cache that are older than
>> +a threshold.
>> +
>> Usage
>> ======================
>> 1. Verify the destination QEMU version is able to decode the new format.
>> diff --git a/include/migration/page_cache.h b/include/migration/page_cache.h
>> index 2d5ce2d..10ed532 100644
>> --- a/include/migration/page_cache.h
>> +++ b/include/migration/page_cache.h
>> @@ -43,8 +43,10 @@ void cache_fini(PageCache *cache);
>>  *
>>  * @cache pointer to the PageCache struct
>>  * @addr: page addr
>> + * @current_age: current bitmap generation
>>  */
>> -bool cache_is_cached(const PageCache *cache, uint64_t addr);
>> +bool cache_is_cached(const PageCache *cache, uint64_t addr,
>> +                     uint64_t current_age);
>> 
>> /**
>>  * get_cached_data: Get the data cached for an addr
>> @@ -60,13 +62,15 @@ uint8_t *get_cached_data(const PageCache *cache, 
>> uint64_t addr);
>>  * cache_insert: insert the page into the cache. the page cache
>>  * will dup the data on insert. the previous value will be overwritten
>>  *
>> - * Returns -1 on error
>> + * Returns -1 when the page isn't inserted into cache
>>  *
>>  * @cache pointer to the PageCache struct
>>  * @addr: page address
>>  * @pdata: pointer to the page
>> + * @current_age: current bitmap generation
>>  */
>> -int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata);
>> +int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata,
>> +                 uint64_t current_age);
>> 
>> /**
>>  * cache_resize: resize the page cache. In case of size reduction the extra
>> diff --git a/page_cache.c b/page_cache.c
>> index b033681..c78157b 100644
>> --- a/page_cache.c
>> +++ b/page_cache.c
>> @@ -33,6 +33,9 @@
>>     do { } while (0)
>> #endif
>> 
>> +/* the page in cache will not be replaced in two cycles */
>> +#define CACHED_PAGE_LIFETIME 2
>> +
>> typedef struct CacheItem CacheItem;
>> 
>> struct CacheItem {
>> @@ -121,7 +124,8 @@ static size_t cache_get_cache_pos(const PageCache *cache,
>>     return pos;
>> }
>> 
>> -bool cache_is_cached(const PageCache *cache, uint64_t addr)
>> +bool cache_is_cached(const PageCache *cache, uint64_t addr,
>> +                     uint64_t current_age)
>> {
>>     size_t pos;
>> 
>> @@ -130,7 +134,12 @@ bool cache_is_cached(const PageCache *cache, uint64_t 
>> addr)
>> 
>>     pos = cache_get_cache_pos(cache, addr);
>> 
>> -    return (cache->page_cache[pos].it_addr == addr);
>> +    if (cache->page_cache[pos].it_addr == addr) {
>> +        /* update the it_age when the cache hit */
>> +        cache->page_cache[pos].it_age = current_age;
>> +        return true;
>> +    }
>> +    return false;
>> }
>> 
>> static CacheItem *cache_get_by_addr(const PageCache *cache, uint64_t addr)
>> @@ -150,7 +159,8 @@ uint8_t *get_cached_data(const PageCache *cache, 
>> uint64_t addr)
>>     return cache_get_by_addr(cache, addr)->it_data;
>> }
>> 
>> -int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata)
>> +int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata,
>> +                 uint64_t current_age)
>> {
>> 
>>     CacheItem *it = NULL;
>> @@ -161,6 +171,11 @@ int cache_insert(PageCache *cache, uint64_t addr, const 
>> uint8_t *pdata)
>>     /* actual update of entry */
>>     it = cache_get_by_addr(cache, addr);
>> 
>> +    if (it->it_data &&
>> +        it->it_age + CACHED_PAGE_LIFETIME > current_age) {
>> +        /* the cache page is fresh, don't replace it */

Hi Dave, is it ok? 

-    if (it->it_data &&
+    if (it->it_data && it->it_addr != addr &&
        it->it_age + CACHED_PAGE_LIFETIME > current_age) {

ChenLiang
>> +        return -1;
>> +    }
>>     /* allocate page */
>>     if (!it->it_data) {
>>         it->it_data = g_try_malloc(cache->page_size);
>> @@ -173,7 +188,7 @@ int cache_insert(PageCache *cache, uint64_t addr, const 
>> uint8_t *pdata)
>> 
>>     memcpy(it->it_data, pdata, cache->page_size);
>> 
>> -    it->it_age = ++cache->max_item_age;
>> +    it->it_age = current_age;
>>     it->it_addr = addr;
>> 
>>     return 0;
>> -- 
>> 1.7.12.4
>> 
>> 
>> 
> --
> Dr. David Alan Gilbert / address@hidden / Manchester, UK
>