Re: [Qemu-devel] [PATCH v5 6/6] BitmapLog: python script for extracting bitmap from a binary file

[Dr. David Alan Gilbert]

* Sanidhya Kashyap (address@hidden) wrote:
> The file not only extracts the bitmap from the file but also draws the figure
> if required. Currently, figure is drawn for all the bitmaps. Later, I'll make
> the change to draw for different blocks.
> 
> The picture is drawn by generating a matrix of 0s and 1s from the bitmap. The
> dimensions are calculated on the basis of total bitmap pages which is 
> represented
> as sqrt(total pages) X (sqrt(total pages) + 1). The white parts indicate non 
> dirtied
> region while the black - dirtied region.
> 
> The python code requires some libraries such as numpy, pylab and math to 
> generate
> the images.

This is interesting; I've tried this out with a Fedora boot and recorded about 
240
frames; the script has problems when trying to convert that big a recording to 
images;
  1) It loads it all into memory - so it used about 2GB of RAM
  2) It opens each image as a window - so I then had to close about 240 windows

I fiddled with it a bit and created .png files, and then stiched these together
with imagemagick to create an animation.  It's quite interesting watch the OS 
boot
and then see me login and start google's stress apptest which suddenly changes
the whole of memory.

I suggest making it dump each image to a file in a similar way and making it 
free the
python image data after each frame, then there shouldn't be a size limit.
(I did try and upload the image, but QEMU's wiki didn't like the size)

Dave


> 
> Signed-off-by: Sanidhya Kashyap <address@hidden>
> ---
>  scripts/extract-bitmap.py | 144 
> ++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 144 insertions(+)
>  create mode 100755 scripts/extract-bitmap.py
> 
> diff --git a/scripts/extract-bitmap.py b/scripts/extract-bitmap.py
> new file mode 100755
> index 0000000..942deca
> --- /dev/null
> +++ b/scripts/extract-bitmap.py
> @@ -0,0 +1,144 @@
> +#!/usr/bin/python
> +# This python script helps in extracting the dirty bitmap present
> +# in the file after executing the log-dirty-bitmap command either
> +# from the qmp or hmp interface. This file only processes binary
> +# file obtained via command.
> +#
> +# Copyright (C) 2014 Sanidhya Kashyap <address@hidden>
> +#
> +# Authors:
> +#       Sanidhya Kashyap
> +#
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2 or later.
> +
> +import struct
> +import argparse
> +from functools import partial
> +from math import sqrt
> +from numpy import array
> +from pylab import figure,imshow,show,gray
> +
> +long_bytes = 8
> +byte_size = 8
> +int_bytes = 4
> +complete_bitmap_list = []
> +block_list = []
> +
> +def get_unsigned_long_integer(value):
> +     return struct.unpack('<Q', value)[0]
> +
> +def get_long_integer(value):
> +     return struct.unpack('<q', value)[0]
> +
> +def get_integer(value):
> +     return struct.unpack('<i', value)[0]
> +
> +def get_char(value):
> +     return struct.unpack('<c', value)[0]
> +
> +def get_string(value, length):
> +     name = struct.unpack('<'+str(length)+'s', value)[0]
> +     for i in range(len(name)):
> +             if name[i] == '\x00':
> +                     return name[:i]
> +
> +def dec2bin(decimal):
> +    bin_value = bin(decimal)[2:]
> +    if len(bin_value) < long_bytes * byte_size:
> +        add_zeroes = long_bytes * byte_size - len(bin_value)
> +        for i in range(add_zeroes):
> +            bin_value += "0"
> +    return str(bin_value)
> +
> +def get_bitmap_length(ram_bitmap_pages):
> +    bitmap_length = ram_bitmap_pages / (long_bytes * byte_size)
> +    if ram_bitmap_pages % (long_bytes * byte_size) != 0:
> +        bitmap_length += 1
> +    return bitmap_length
> +
> +def dump_ram_block_info(infile):
> +    total_blocks = get_integer(infile.read(int_bytes))
> +    for i in range(total_blocks):
> +        block_name_length = get_integer(infile.read(int_bytes))
> +        block_name = get_string(infile.read(block_name_length), 
> block_name_length)
> +        block_offset = get_unsigned_long_integer(infile.read(long_bytes))
> +        block_length = get_unsigned_long_integer(infile.read(long_bytes))
> +        block_list.append(dict(name=block_name, offset=block_offset, 
> length=block_length))
> +
> +def generate_images():
> +    r = 0
> +    for list in complete_bitmap_list:
> +        all_digits = ""
> +        for element in list:
> +            all_digits += dec2bin(element)
> +        l = len(all_digits)
> +        sqrtvalue = int(sqrt(l))
> +        for i in range(sqrtvalue * (sqrtvalue+1)-l):
> +            all_digits+="0"
> +
> +        v = []
> +        l = len(all_digits)
> +        for i in range(sqrtvalue+1):
> +            v1 = []
> +            for j in range(sqrtvalue):
> +                v1.append(int(all_digits[i*sqrtvalue+j]))
> +            v.append(v1)
> +
> +        im_array = array(v)
> +        figure(r)
> +        imshow(im_array, cmap=gray())
> +        r += 1
> +    show()
> +
> +def dump_bitmap(infile, draw):
> +    marker = 'M'
> +    count = 0
> +    value = ' '
> +    current_ram_bitmap_pages = 0
> +    prev_ram_bitmap_pages = 0
> +    while True:
> +        if len(value) == 0  or marker != 'M':
> +            print "issue with the dump"
> +            return
> +        bitmap_page_raw_value = infile.read(long_bytes)
> +        if not bitmap_page_raw_value:
> +            break
> +        current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value)
> +        if current_ram_bitmap_pages != prev_ram_bitmap_pages:
> +            prev_ram_bitmap_pages = current_ram_bitmap_pages
> +            dump_ram_block_info(infile)
> +
> +        bitmap_length = get_bitmap_length(current_ram_bitmap_pages)
> +        bitmap_list = []
> +        bitmap_raw_value = infile.read(long_bytes * bitmap_length)
> +        if not bitmap_raw_value:
> +            break
> +        count+=1
> +        for i in range(bitmap_length):
> +            mark = i * long_bytes
> +            
> bitmap_list.append((get_unsigned_long_integer(bitmap_raw_value[mark:mark+long_bytes])))
> +        complete_bitmap_list.append(bitmap_list)
> +        value = infile.read(1)
> +        marker = get_char(value)
> +    if draw is True:
> +        generate_images()
> +    else:
> +        print complete_bitmap_list
> +
> +def main():
> +    extracter = argparse.ArgumentParser(description='Extract dirty bitmap 
> from binary file.')
> +    extracter.add_argument('infile', help='Input file to extract the bitmap')
> +    extracter.add_argument('-d', action='store_true', dest='draw', 
> default=False,
> +            help='Draw a black and white image of the processed dirty 
> bitmap')
> +    args = extracter.parse_args()
> +    print 'The filename is {}'.format(args.infile)
> +
> +    infile = open(format(args.infile), 'rb')
> +
> +    dump_bitmap(infile, args.draw);
> +
> +    infile.close()
> +
> +if __name__ == '__main__':
> +    main()
> -- 
> 1.9.3
> 
> 
--
Dr. David Alan Gilbert / address@hidden / Manchester, UK