[gnunet] 70/164: Rewritten dynamic counter

[gnunet]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a commit to branch master
in repository gnunet.

commit f2c7cfa470208029eaa8a5e7cf15eadd3fea6677
Author: Elias Summermatter <elias.summermatter@seccom.ch>
AuthorDate: Wed Apr 28 21:23:28 2021 +0200

    Rewritten dynamic counter
---
 src/setu/gnunet-service-setu.c |   6 ++
 src/setu/ibf.c                 | 230 ++++++++++++++++++++++++++++++++++++-----
 src/setu/perf_setu_api.c       |   6 +-
 3 files changed, 213 insertions(+), 29 deletions(-)

diff --git a/src/setu/gnunet-service-setu.c b/src/setu/gnunet-service-setu.c
index 60d8095a4..d029838d7 100644
--- a/src/setu/gnunet-service-setu.c
+++ b/src/setu/gnunet-service-setu.c
@@ -719,6 +719,7 @@ uint32_t
 calculate_min_buckets(struct Operation *op)
 {
     // Save local set size
+    /**
     if(0 == op->local_element_count) {
         op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
                 op->set->content->elements);
@@ -728,6 +729,7 @@ calculate_min_buckets(struct Operation *op)
     if(calculated_min >= IBF_MIN_SIZE) {
         return calculated_min | 1;
     }
+     **/
     return IBF_MIN_SIZE | 1;
 }
 
@@ -1461,6 +1463,10 @@ send_ibf (struct Operation *op,
     msg->offset = htonl (buckets_sent);
     msg->salt = htonl (op->salt_send);
     msg->ibf_counter_bit_length = ibf_get_max_counter(ibf);
+
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           "HERE: %d\n", msg->ibf_counter_bit_length);
+
     ibf_write_slice (ibf, buckets_sent,
                      buckets_in_message, &msg[1], msg->ibf_counter_bit_length);
     buckets_sent += buckets_in_message;
diff --git a/src/setu/ibf.c b/src/setu/ibf.c
index 27d053cd1..16a1d0147 100644
--- a/src/setu/ibf.c
+++ b/src/setu/ibf.c
@@ -306,13 +306,13 @@ ibf_decode (struct InvertibleBloomFilter *ibf,
 uint8_t
 ibf_get_max_counter (struct InvertibleBloomFilter *ibf)
     {
-    uint64_t max_counter=8;
+    uint64_t max_counter=0;
     for (uint64_t i = 0; i < ibf->size; i++) {
         if(ibf->count[i].count_val > max_counter){
             max_counter=ibf->count[i].count_val;
         }
     }
-        return floor(log2(max_counter)) + 1;
+    return floor(log2(max_counter)) + 1;
 
 }
 
@@ -352,11 +352,11 @@ ibf_write_slice (const struct InvertibleBloomFilter *ibf,
   key_hash_dst += count;
 
   /* pack and copy counter */
-  size_t buf_size = ceil((float ) count/(8/(float)counter_max_length));
+  size_t buf_size = ceil((float) count/(8/(float)counter_max_length));
   uint64_t *new_buf;
-  new_buf = (uint64_t*) GNUNET_malloc(buf_size);
+  new_buf = (uint8_t*) GNUNET_malloc(buf_size);
   pack_counter(ibf,start,count,new_buf,counter_max_length);
-  count_dst = (uint64_t *) key_hash_dst;
+  count_dst = (uint8_t *) key_hash_dst;
   GNUNET_memcpy (count_dst,
                    new_buf,
                    buf_size);
@@ -377,25 +377,111 @@ void
 pack_counter(const struct InvertibleBloomFilter *ibf,
              uint32_t start,
              uint64_t count,
-             uint64_t *buf,
+             uint8_t *buf,
              uint8_t counter_max_length)
 {
-    uint64_t buf_position=0;
-    for (uint32_t i = start; i< (count + start);){
-        uint64_t byteToWrite = 0x0;
-        uint32_t bytes_to_shift = ((sizeof(uint64_t) * 8)/ counter_max_length);
-
-        for (uint32_t l = 0; l < bytes_to_shift; l++) {
-            if(i>=count) {
-                byteToWrite = byteToWrite << (counter_max_length * 
(bytes_to_shift - l));
-                break;
+    uint8_t store_size=0;
+    uint8_t store=0;
+    uint16_t byte_ctr=0;
+
+    for (uint64_t i = start; i< (count + start);) {
+            uint64_t count_val_to_write = ibf->count[i].count_val;
+            uint8_t count_len_to_write = counter_max_length;
+            /**
+            LOG (GNUNET_ERROR_TYPE_ERROR,
+                 "NEW VAL: count_val_to_write: %d count_len_to_write:%u \n", 
count_val_to_write, count_len_to_write);
+                 **/
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "SAVE VALUE: %X\n",
+             count_val_to_write);
+            while(count_len_to_write > 0) {
+                uint8_t byte_to_write=0;
+
+                if((count_len_to_write + store_size) >= 8) {
+                    uint8_t bit_shift = 0;
+                    if(store_size > 0 || count_len_to_write > 8) {
+                        uint8_t bit_unused = 8 - store_size;
+                        bit_shift = count_len_to_write - bit_unused;
+                        store = store << bit_unused;
+                    }
+                    /**
+                    LOG (GNUNET_ERROR_TYPE_ERROR,
+                         "DEBUG GT 4 S: byte_to_write: %X, count_val_to_write: 
%016Xq count_len_to_write: %u, store_size: %u store: %X, bit_shift: %d \n",
+                         byte_to_write,
+                         count_val_to_write,
+                         count_len_to_write,
+                         store_size,
+                         store,
+                         bit_shift
+                    ); **/
+
+                    byte_to_write = (( count_val_to_write >> bit_shift) | 
store) & 0xFF;
+                    count_len_to_write -= (8 - store_size);
+                    count_val_to_write = count_val_to_write & (( 1ULL << 
count_len_to_write ) - 1);
+                    /**
+                    LOG (GNUNET_ERROR_TYPE_ERROR,
+                         "DEBUG GT 4 E: byte_to_write: %X, count_val_to_write: 
%016X count_len_to_write: %u, store_size: %u store: %X  and %X\n",
+                         byte_to_write,
+                         count_val_to_write,
+                         count_len_to_write,
+                         store_size,
+                         store,
+                         (( 1 << count_len_to_write ) - 1)
+                    ); **/
+
+                    store=0;
+                    store_size=0;
+                } else {
+                    /**
+                    LOG (GNUNET_ERROR_TYPE_ERROR,
+                         "DEBUG STORE S: byte_to_write: %X, 
count_val_to_write: %x count_len_to_write: %u, store_size: %u store: %X\n",
+                         byte_to_write,
+                         count_val_to_write,
+                         count_len_to_write,
+                         store_size,
+                         store
+                    );
+                    **/
+                    if(0 == store){
+                        store = count_val_to_write;
+                    } else {
+                        store = (store << count_len_to_write) | 
count_val_to_write ;
+                    }
+                    store_size = store_size + count_len_to_write;
+                    count_len_to_write -= count_len_to_write;
+                    /**
+                    LOG (GNUNET_ERROR_TYPE_ERROR,
+                         "DEBUG STORE E: byte_to_write: %X, 
count_val_to_write: %x count_len_to_write: %u, store_size: %u store: %X\n",
+                         byte_to_write,
+                         count_val_to_write,
+                         count_len_to_write,
+                         store_size,
+                         store
+                    ); **/
+
+                    break;
+                }
+
+                buf[byte_ctr] = byte_to_write;
+                /**
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "PACKED %u -> %X\n",
+                     ibf->count[i].count_val,
+                     byte_to_write); **/
+                byte_ctr++;
             }
-            byteToWrite = (byteToWrite << counter_max_length) | 
ibf->count[i].count_val;
             i++;
-        }
-        buf[buf_position] = byteToWrite;
-        buf_position++;
     }
+
+    /**
+    LOG (GNUNET_ERROR_TYPE_ERROR,
+         "LAST PACKED STORE_ORG: %X, STORE_SH: %x size: %d \n",
+         store,
+         store << (8 - store_size),
+         store_size);  **/
+    buf[byte_ctr] = store << (8 - store_size);
+    byte_ctr++;
+
 }
 
 /**
@@ -412,20 +498,108 @@ void
 unpack_counter(const struct InvertibleBloomFilter *ibf,
                uint32_t start,
                uint64_t count,
-               uint64_t *buf,
+               uint8_t *buf,
                uint8_t counter_max_length)
 {
-    uint32_t element_counter=0;
-    for (uint32_t i = 0; i < ceil((float ) 
count/(64/(float)counter_max_length)); i++){
-        for(uint32_t l = (64/counter_max_length); l > 0; l-- ) {
-            uint64_t value = 0;
-            value = (buf[i] >> (counter_max_length * (l -1))) & ((uint64_t) 
pow(2,counter_max_length) - 1 );
+    uint64_t ibf_counter_ctr = 0;
+    uint64_t store = 0;
+    uint64_t store_bit_ctr = 0;
+    uint64_t byte_ctr = 0;
+
+    uint64_t number_bytes_read = ceil(( count * counter_max_length) / 8);
+    while (ibf_counter_ctr <= (count - 1)){
+        uint8_t byte_read = buf[byte_ctr];
+        uint8_t bit_to_read_left = 8;
+        byte_ctr++;
+        /**
+        LOG (GNUNET_ERROR_TYPE_ERROR, "DECODED_BYTE: %X\n",byte_read);
+        **/
+        while (bit_to_read_left >= 0) {
+
+            if(ibf_counter_ctr > (count - 1))
+                return;
+
+            /**
+            LOG (GNUNET_ERROR_TYPE_ERROR, "CTR: ibf_counter_ctr: %d count: 
%d\n", ibf_counter_ctr,count);
+            **/
+
+            if ((store_bit_ctr + bit_to_read_left) >= counter_max_length) {
+                /**
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "GT 8 START: byte_read: %X, store: %X store_bit_ctr: %u 
bit_to_read_left: %u \n",
+                     byte_read,
+                     store,
+                     store_bit_ctr,
+                     bit_to_read_left
+                     );**/
+                uint8_t bytes_used = counter_max_length - store_bit_ctr;
+                if(store_bit_ctr > 0) {
+                    store = store << bytes_used;
+                }
+
+                uint8_t bytes_to_shift = bit_to_read_left - bytes_used;
+                uint64_t counter_part = byte_read >> bytes_to_shift;
+                store=store | counter_part;
+                ibf->count[ibf_counter_ctr].count_val = store;
+
+                byte_read = byte_read & (( 1 << bytes_to_shift ) - 1);
+                /**
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "GT 8 END: byte_read: %X, store: %X store_bit_ctr: %u 
bit_to_read_left: %u ibf_counter_ctr: %u bytes_used: %u counter_part: %x \n",
+                     byte_read,
+                     store,
+                     store_bit_ctr,
+                     bit_to_read_left,
+                     ibf_counter_ctr,
+                     bytes_used,
+                     counter_part
+                ); **/
+
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "RESULT: %u\n",
+                     ibf->count[ibf_counter_ctr].count_val
+                );
+
+                bit_to_read_left -= bytes_used;
+                ibf_counter_ctr++;
+                store=0;
+                store_bit_ctr=0;
+
+
+
+            } else {
+                /**
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "STORE START: byte_read: %X, store: %X store_bit_ctr: %u 
\n",
+                     byte_read,
+                     store,
+                     store_bit_ctr);
+                **/
+                store_bit_ctr += bit_to_read_left;
+                if(0 == store) {
+                    store=byte_read;
+                } else {
+                    store=store << bit_to_read_left;
+                    store=store | byte_read;
+                }
+                /**
+                LOG (GNUNET_ERROR_TYPE_ERROR,
+                     "STORE END: byte_read: %X, store: %X store_bit_ctr: %u 
\n",
+                     byte_read,
+                     store,
+                     store_bit_ctr);
+**/
+                break;
 
-            if(element_counter>(ibf->size -1 )) return;
+            }
 
-            ibf->count[element_counter].count_val = value;
-            element_counter++;
         }
+
+
+        /**
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "UNPACKED %X\n",
+             byte_read);**/
     }
 
 }
diff --git a/src/setu/perf_setu_api.c b/src/setu/perf_setu_api.c
index fea836765..7d605e42e 100644
--- a/src/setu/perf_setu_api.c
+++ b/src/setu/perf_setu_api.c
@@ -404,7 +404,7 @@ run (void *cls,
                 "Running real set-reconciliation\n");
     //init_set1 ();
     // limit ~23800 element total
-    initRandomSets(0,45,50,32);
+    initRandomSets(4998, 5000,5000,32);
 }
 
 void perf_thread() {
@@ -425,6 +425,9 @@ static void run_petf_thread(int total_runs) {
 //Father code (before child processes start)
     for (int processed = 0; processed < total_runs;) {
         for (int id = 0; id < core_count; id++) {
+            perf_thread();
+        }
+            /**
             if(processed >= total_runs) break;
 
             if ((child_pid = fork()) == 0) {
@@ -434,6 +437,7 @@ static void run_petf_thread(int total_runs) {
             processed += 1;
         }
         while ((wpid = wait(&status)) > 0);
+             **/
     }
 }
 

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.