[Commit-gnuradio] r7015 - in gnuradio/branches/developers/eb/gcell/src: include lib

[eb]

Author: eb
Date: 2007-11-21 19:32:54 -0700 (Wed, 21 Nov 2007)
New Revision: 7015

Modified:
   gnuradio/branches/developers/eb/gcell/src/include/compiler.h
   gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc.h
   gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc_private.h
   gnuradio/branches/developers/eb/gcell/src/lib/gc_client_thread_info.h
   gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.cc
   gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.h
Log:
create cache-aligned data structures

Modified: gnuradio/branches/developers/eb/gcell/src/include/compiler.h
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/include/compiler.h        
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/include/compiler.h        
2007-11-22 02:32:54 UTC (rev 7015)
@@ -26,6 +26,7 @@
  * \brief Compiler specific hackery.  These are for GCC.
  */
 
+#define _AL8   __attribute__((aligned (8)))
 #define _AL16  __attribute__((aligned (16)))
 #define _AL128 __attribute__((aligned (128)))
 

Modified: gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc.h
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc.h     
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc.h     
2007-11-22 02:32:54 UTC (rev 7015)
@@ -101,10 +101,10 @@
   uint64_t      u64;
   float                 f;
   double        d;
-  //float complex       fc;    //  64-bits (C99)
-  //double complex dc;    // 128-bits (C99)
+  //float complex  fc; //  64-bits (C99)
+  //double complex dc;  // 128-bits (C99)
   gc_eaddr_t    ea;    //  64-bits
-} _AL16 gc_arg_union_t;
+} _AL8 gc_arg_union_t;
 
 
 /*!

Modified: 
gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc_private.h
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc_private.h     
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/include/gc_job_desc_private.h     
2007-11-22 02:32:54 UTC (rev 7015)
@@ -29,12 +29,12 @@
  */
 typedef struct gc_job_desc_private
 {
-  gc_eaddr_t   next;           // used to implement job queue and free list
+  volatile gc_eaddr_t  next;           // used to implement job queue and free 
list
+  uint16_t             job_id;
 
   // FIXME:
   // notifier_method
   // notifier_arg
-  // location (where we're running)
   
 } gc_job_desc_private_t;
 

Modified: gnuradio/branches/developers/eb/gcell/src/lib/gc_client_thread_info.h
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/lib/gc_client_thread_info.h       
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/lib/gc_client_thread_info.h       
2007-11-22 02:32:54 UTC (rev 7015)
@@ -22,7 +22,7 @@
 #define INCLUDED_GC_CLIENT_THREAD_INFO_H
 
 #include <omnithread.h>
-#include <gc_atomic.h>
+#include <boost/utility.hpp>
 
 enum gc_ct_state_t {
   CT_RUNNING,
@@ -38,20 +38,22 @@
  * store a pointer to one of these for each thread
  * that comes our way.
  */
-class gc_client_thread_info {
+class gc_client_thread_info : boost::noncopyable {
 public:
   gc_client_thread_info() :
-    d_free(1), d_state(CT_RUNNING), d_cond(&d_mutex) { }
+    d_free(1), d_state(CT_RUNNING), d_cond(&d_mutex),
+    d_jobs_were_waiting_for(0){ }
 
   ~gc_client_thread_info() {
     d_free = 1;
+    d_jobs_were_waiting_for = 0;
   }
 
   volatile uint32_t    d_free;         // is this cti free? (1->free, 0->in 
use)
   gc_ct_state_t                d_state;
   omni_mutex           d_mutex;        // used to suspend/resume client
   omni_condition       d_cond;         // used to suspend/resume client
-  unsigned long                d_jobs_were_waiting_for[0];  // bitvector
+  unsigned long               *d_jobs_were_waiting_for;  // bitvector
 };
 
 #endif /* INCLUDED_GC_CLIENT_THREAD_INFO_H */

Modified: gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.cc
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.cc        
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.cc        
2007-11-22 02:32:54 UTC (rev 7015)
@@ -26,15 +26,17 @@
 
 #include <stdio.h>
 #include <stdexcept>
+#include <stdlib.h>
 
 static const bool VERBOSE = true;
 
-static const unsigned int DEFAULT_MAX_JOBS = 256;
+static const size_t CACHE_LINE_SIZE = 128;
+static const unsigned int DEFAULT_MAX_JOBS = 128;
 static const unsigned int DEFAULT_MAX_CLIENT_THREADS = 64;
 
 
 // custom deleter of gang_contexts for use with boost::shared_ptr
-class gang_destroyer {
+class gang_deleter {
 public:
   void operator()(spe_gang_context_ptr_t ctx) {
     if (ctx){
@@ -46,9 +48,17 @@
   }
 };
 
+// custom deleter of anything that can be freed with "free"
+class free_deleter {
+public:
+  void operator()(void *p) {
+    free(p);
+  }
+};
 
+
 gc_job_manager_impl::gc_job_manager_impl(const gc_jm_options *options)
-  : d_nclients(0)
+  : d_nclients(0), d_client_thread(0), d_job_busy(0)
 {
   if (options != 0)
     d_options = *options;
@@ -102,7 +112,7 @@
   }
 
   if (d_options.gang_schedule){
-    d_gang = spe_gang_context_sptr(spe_gang_context_create(0), 
gang_destroyer());
+    d_gang = spe_gang_context_sptr(spe_gang_context_create(0), gang_deleter());
     if (!d_gang){
       perror("gc_job_manager_impl[spe_gang_context_create]");
       throw std::runtime_error("spe_gang_context_create");
@@ -125,13 +135,80 @@
     // FIXME load some code in the SPE
   }
 
-  // FIXME init d_jd
-  // FIXME init d_bvlen
-  // FIXME init d_job_busy
-  // FIXME init d_client_thread
 
+  // ----------------------------------------------------------------
+  // initalize the free list of job descriptors
   gc_jd_stack_init(&d_free_list);
 
+  printf("sizeof(d_jd[0]) = %d (0x%x)\n", sizeof(d_jd[0]), sizeof(d_jd[0]));
+  printf("max_jobs = %u\n", d_options.max_jobs);
+
+  // Initialize the array of job descriptors.
+  void *p = 0;
+  if (posix_memalign(&p, CACHE_LINE_SIZE,
+                    sizeof(d_jd[0]) * d_options.max_jobs) != 0){
+    perror("posix_memalign");
+    throw std::runtime_error("memory");
+  }
+  d_jd = (gc_job_desc_t *) p;
+  memset(d_jd, 0, sizeof(d_jd[0]) * d_options.max_jobs);
+
+  // This ensures that the memory associated with d_jd is
+  // automatically freed in the destructor or if an exception occurs
+  // here in the constructor.  It is admittedly a somewhat strange
+  // use; however we need the the custom deleter, which is only
+  // available with shared_ptr.
+  _d_jd_boost = boost::shared_ptr<void>((void *) d_jd, free_deleter());
+
+
+  // set unique job_id
+  for (int i = 0; i < (int) d_options.max_jobs; i++)
+    d_jd[i].sys.job_id = i;
+
+  // push them onto the free list
+  for (int i = d_options.max_jobs - 1; i >= 0; i--)
+    free_job_desc(&d_jd[i]);
+
+
+  // ----------------------------------------------------------------
+  // initialize d_client_thread
+
+  // FIXME robust deletion
+  d_client_thread = new gc_client_thread_info[d_options.max_client_threads];
+  
+  // ----------------------------------------------------------------
+  // initialize bitvectors
+
+  // initialize d_bvlen, the number of longs in job related bitvectors.
+  int bits_per_long = sizeof(unsigned long) * 8;
+  d_bvlen = (d_options.max_jobs + bits_per_long - 1) / bits_per_long;
+
+  // allocate all bitvectors in a single cache-aligned chunk
+  size_t nlongs = (1 + d_options.max_client_threads) * d_bvlen;
+  size_t byte_size = nlongs * sizeof(unsigned long);
+  p = 0;
+  if (posix_memalign(&p, CACHE_LINE_SIZE, byte_size) != 0){
+    perror("posix_memalign");
+    throw std::runtime_error("memory");
+  }
+  memset(p, 0, byte_size);
+
+  // let boost keep track of the storage
+  _d_all_bitvectors = boost::shared_ptr<void>(p, free_deleter());
+
+  // now point the other pointers into this storage.
+  // d_job_busy is the first one, followed by the ones in the client data
+
+  unsigned long *v = (unsigned long *) p;
+  d_job_busy = v;
+  v += d_bvlen;
+
+  for (unsigned int i = 0; i < d_options.max_client_threads; i++, v += d_bvlen)
+    d_client_thread[i].d_jobs_were_waiting_for = v;
+
+  // ----------------------------------------------------------------
+
+
   // FIXME sequence this...
   // FIXME create event handler thread
   // FIXME create N worker threads
@@ -145,6 +222,12 @@
   shutdown();
 
   // FIXME whatever else needs to be done
+
+  delete [] d_client_thread;
+  d_client_thread = 0;
+
+  d_jd = 0;            // handled via _d_jd_boost
+  d_job_busy = 0;      // handled via _d_all_bitvectors
 }
 
 bool

Modified: gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.h
===================================================================
--- gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.h 
2007-11-21 23:39:04 UTC (rev 7014)
+++ gnuradio/branches/developers/eb/gcell/src/lib/gc_job_manager_impl.h 
2007-11-22 02:32:54 UTC (rev 7015)
@@ -26,10 +26,12 @@
 #include "gc_client_thread_info.h"
 #include "gc_jd_stack.h"
 #include <libspe2.h>
+#include <vector>
 #include <boost/shared_ptr.hpp>
 
 typedef boost::shared_ptr<spe_gang_context> spe_gang_context_sptr;
 
+
 enum worker_state {
   WS_FREE,     // not in use
   WS_INIT,     // allocated and being initialized
@@ -58,14 +60,30 @@
   spe_gang_context_sptr  d_gang;               // boost::shared_ptr
   worker_ctx            d_worker[MAX_SPES];
 
-  gc_job_desc_t                *d_jd;                  // [limits.max_jobs]
+  // All of the job descriptors are hung off of here.
+  // We allocate them all in a single cache aligned chunk.
+  gc_job_desc_t                *d_jd;                  // [options.max_jobs]
+  boost::shared_ptr<void> _d_jd_boost;         // hack for automatic storage 
mgmt
 
-  int                   d_nclients;            // # of used entries in 
d_client_thread
-  gc_client_thread_info        *d_client_thread;       // 
[limits.max_client_threads]
+  int                   d_nclients;            // active length of 
d_client_thread
+  gc_client_thread_info *d_client_thread;      // [options.max_client_threads]
 
+  // We use bitvectors to represent the busy/free state of a job.  Each
+  // bitvector is d_bvlen longs in length.
   int                   d_bvlen;               // bit vector length in longs
-  unsigned long                *d_job_busy;            // bitvector of job's 
status 1=BUSY
 
+  // This contains the storage for all the bitvectors used by the job
+  // manager.  There's 1 for d_job_busy and 1 for each client thread
+  // [options.max_client_threads].  We allocate them all in a single
+  // cache aligned chunk.
+  boost::shared_ptr<void> _d_all_bitvectors;   // hack for automatic storage 
mgmt
+
+  // bitvector that describes that state of all jobs (1=BUSY).
+  // This points into _d_all_bitvectors.
+  unsigned long                *d_job_busy;
+
+  // The actually allocation chunk is hung off of d_jd.
+  // This is the lock free stack where we keep track of the free ones.
   gc_jd_stack_t                 d_free_list;           // stack of free job 
descriptors