Re: [Qemu-devel] [PATCH 3/3] layout: Add generators for refcount tbles and blocks

[M.Kustova]

On Tue, Aug 19, 2014 at 1:36 PM, Fam Zheng <address@hidden> wrote:
> On Mon, 08/11 15:55, Maria Kustova wrote:
>> Refcount structures are placed in clusters randomly selected from all not
>> allocated host clusters.
>
> s/not allocated/unallocated/
>
>>
>> Signed-off-by: Maria Kustova <address@hidden>
>> ---
>>  tests/image-fuzzer/qcow2/layout.py | 136 
>> ++++++++++++++++++++++++++++++++++++-
>>  1 file changed, 135 insertions(+), 1 deletion(-)
>>
>> diff --git a/tests/image-fuzzer/qcow2/layout.py 
>> b/tests/image-fuzzer/qcow2/layout.py
>> index 730c771..2239789 100644
>> --- a/tests/image-fuzzer/qcow2/layout.py
>> +++ b/tests/image-fuzzer/qcow2/layout.py
>> @@ -102,6 +102,8 @@ class Image(object):
>>          self.end_of_extension_area = FieldsList()
>>          self.l2_tables = FieldsList()
>>          self.l1_table = FieldsList()
>> +        self.refcount_table = FieldsList()
>> +        self.refcount_blocks = FieldsList()
>>          self.ext_offset = 0
>>          self.create_header(cluster_bits, backing_file_name)
>>          self.set_backing_file_name(backing_file_name)
>> @@ -113,7 +115,8 @@ class Image(object):
>>      def __iter__(self):
>>          return chain(self.header, self.backing_file_format,
>>                       self.feature_name_table, self.end_of_extension_area,
>> -                     self.backing_file_name, self.l1_table, self.l2_tables)
>> +                     self.backing_file_name, self.l1_table, self.l2_tables,
>> +                     self.refcount_table, self.refcount_blocks)
>>
>>      def create_header(self, cluster_bits, backing_file_name=None):
>>          """Generate a random valid header."""
>> @@ -330,6 +333,136 @@ class Image(object):
>>                                                  
>> float(self.cluster_size**2)))
>>          self.header['l1_table_offset'][0].value = l1_offset
>>
>> +    def create_refcount_structures(self):
>> +        """Generate random refcount blocks and refcount table."""
>> +        def allocate_rfc_blocks(data, size):
>> +            """Return indices of clusters allocated for recount blocks."""
>
> s/recount/refcount/
>
>> +            cluster_ids = set()
>> +            diff = block_ids = set([x / size for x in data])
>> +            while len(diff) != 0:
>> +                # Allocate all yet not allocated clusters
>> +                new = self._get_available_clusters(data | cluster_ids,
>> +                                                   len(diff))
>> +                # Indices of new refcount blocks necessary to cover clusters
>> +                # in 'new'
>> +                diff = set([x / size for x in new]) - block_ids
>> +                cluster_ids |= new
>> +                block_ids |= diff
>> +            return cluster_ids, block_ids
>> +
>> +        def allocate_rfc_table(data, init_blocks, block_size):
>> +            """Return indices of clusters allocated for the refcount table
>> +            and updated indices of clusters allocated for blocks and indices
>> +            of blocks.
>> +            """
>> +            blocks = set(init_blocks)
>> +            clusters = set()
>> +            # Number of entries in one cluster of the refcount table
>> +            size = self.cluster_size / UINT64_S
>> +            # Number of clusters necessary for the refcount table based on
>> +            # the current number of refcount blocks
>> +            table_size = int(ceil((max(blocks) + 1) / float(size)))
>> +            # Index of the first cluster of the refcount table
>> +            table_start = self._get_adjacent_clusters(data, table_size + 1)
>> +            # Clusters allocated for the current length of the refcount 
>> table
>> +            table_clusters = set(range(table_start, table_start + 
>> table_size))
>> +            # Clusters allocated for the refcount table including
>> +            # last optional one for potential l1 growth
>> +            table_clusters_allocated = set(range(table_start, table_start +
>> +                                                 table_size + 1))
>> +            # New refcount blocks necessary for clusters occupied by the
>> +            # refcount table
>> +            diff = set([c / block_size for c in table_clusters]) - blocks
>> +            blocks |= diff
>> +            while len(diff) != 0:
>> +                # Allocate clusters for new refcount blocks
>> +                new = self._get_available_clusters((data | clusters) |
>> +                                                   table_clusters_allocated,
>> +                                                   len(diff))
>> +                # Indices of new refcount blocks necessary to cover
>> +                # clusters in 'new'
>> +                diff = set([x / block_size for x in new]) - blocks
>> +                clusters |= new
>> +                blocks |= diff
>> +                # Check if the refcount table needs one more cluster
>> +                if int(ceil((max(blocks) + 1) / float(size))) > table_size:
>> +                    new_block_id = (table_start + table_size) / block_size
>> +                    # Check if the additional table cluster needs
>> +                    # one more refcount block
>> +                    if new_block_id not in blocks:
>> +                        diff.add(new_block_id)
>> +                    table_clusters.add(table_start + table_size)
>> +                    table_size += 1
>> +            return table_clusters, blocks, clusters
>> +
>> +        def create_table_entry(table_offset, block_cluster, block_size,
>> +                               cluster):
>> +            """Generate a refcount table entry."""
>> +            offset = table_offset + UINT64_S * (cluster / block_size)
>> +            return ['>Q', offset, block_cluster * self.cluster_size,
>> +                    'refcount_table_entry']
>> +
>> +        def create_block_entry(block_cluster, block_size, cluster):
>> +            """Generate a list of entries for the current block."""
>> +            entry_size = self.cluster_size / block_size
>> +            offset = block_cluster * self.cluster_size
>> +            entry_offset = offset + entry_size * (cluster % block_size)
>> +            # While snapshots are not supported all refcounts are set to 1
>> +            return ['>H',entry_offset, 1, 'refcount_block_entry']
>> +
>> +        # Number of refcount entries per refcount block
>> +        block_size = self.cluster_size / \
>> +                     (1 << self.header['refcount_order'][0].value - 3)
>
> Why minus 3? Could you use a named constant or a comment?
   Conversion to bytes. I will make it more descriptive.