[GNUnet-SVN] r24330 - monkey/trunk/seaspider

[gnunet]

Author: teichm
Date: 2012-10-15 23:01:46 +0200 (Mon, 15 Oct 2012)
New Revision: 24330

Modified:
   monkey/trunk/seaspider/seasp2
   monkey/trunk/seaspider/seasp2_convert
Log:
seasp2_convert can now also traverse a folder
basic SystemGlobal support


Modified: monkey/trunk/seaspider/seasp2
===================================================================
--- monkey/trunk/seaspider/seasp2       2012-10-15 20:24:46 UTC (rev 24329)
+++ monkey/trunk/seaspider/seasp2       2012-10-15 21:01:46 UTC (rev 24330)
@@ -16,6 +16,5 @@
 export CFLAGS="--seaspider -m32"
 ./configure
 make $1
-cat `find * -name "*.sea"` > expressions.sea
-# probably call seasp2_convert here later
-
+seasp2_convert -o expressions.sqlite
+rm -f `find * -name "*.sea"`

Modified: monkey/trunk/seaspider/seasp2_convert
===================================================================
--- monkey/trunk/seaspider/seasp2_convert       2012-10-15 20:24:46 UTC (rev 
24329)
+++ monkey/trunk/seaspider/seasp2_convert       2012-10-15 21:01:46 UTC (rev 
24330)
@@ -5,16 +5,40 @@
 import os
 from collections import deque
 
+
+# argparse Action for a readable dir
+class readable_dir(argparse.Action):
+       def __call__(self, parser, namespace, values, option_string=None):
+               prospective_dir=values
+               if not os.path.exists(prospective_dir):
+                       raise argparse.ArgumentTypeError("readable_dir:{0} is 
not a valid path".format(prospective_dir))
+               if os.access(prospective_dir, os.R_OK):
+                       setattr(namespace, self.dest, prospective_dir)
+               else:
+                       raise argparse.ArgumentTypeError("readable_dir:{0} is 
not a readable dir".format(prospective_dir))
+
+
+# initialize argument parser
 parser = argparse.ArgumentParser(description="converts .sea files generated by 
cparser to a seaspider sqlite database")
 
 parser.add_argument('--version', action='version', version='db_convert 1.0')
-parser.add_argument("-o", "--output", metavar="FILENAME", 
type=argparse.FileType('w'), dest="outfile", help="filename for the resulting 
sqlite database. defaulting to infile.sqlite")
-parser.add_argument('infile', type=argparse.FileType('r'), help='input file')
+parser.add_argument("-o", "--output", metavar="FILENAME", 
type=argparse.FileType('w'), dest="outfile", help="filename for the resulting 
sqlite database. defaults to INPUT.sqlite")
+parser.add_argument('INPUT', nargs='?', action=readable_dir, 
default=os.getcwd(), help='input file or directory which is traversed 
recursively. defaults to PWD')
 args = parser.parse_args()
 
+
+# build output filename
 if args.outfile== None:
-       (outfilename, ext) = os.path.splitext(args.infile.name)
-       outfilename += '.sqlite'
+       if os.path.isdir(args.INPUT):
+               in_dir = os.path.abspath(args.INPUT)
+               (crap, tmp_name) = os.path.split(in_dir)
+               outfilename = in_dir + '/' + tmp_name + '.sqlite'
+       elif os.path.isfile(args.INPUT):
+               (outfilename, ext) = os.path.splitext(args.INPUT)
+               outfilename += '.sqlite'
+       else:
+               print 'unknown output target, writing to expressions.sqlite'
+               outfilename = 'expressions.sqlite'
 else:
        outfilename = args.outfile.name
        args.outfile.close()
@@ -25,73 +49,110 @@
 except OSError:
        pass
 
-
 # connect to db
-conn = sqlite3.connect(outfilename)
-c = conn.cursor()
+sql_conn = sqlite3.connect(outfilename)
+sql_cur = sql_conn.cursor()
 
 # create table
-c.execute('''CREATE TABLE Expression (
-                       expr_ID INTEGER PRIMARY KEY AUTOINCREMENT,
-                       file_name TEXT NOT NULL,
-                       expr_syntax TEXT,
-                       start_lineno INT,
-                       end_lineno INT,
-                       is_call INT)''')
+sql_cur.execute('''CREATE TABLE Expression (
+                               expr_ID INTEGER PRIMARY KEY AUTOINCREMENT,
+                               file_name TEXT NOT NULL,
+                               expr_syntax TEXT,
+                               start_lineno INT,
+                               end_lineno INT,
+                               is_call INT)''')
 
-# commit changes
-conn.commit()
 
-scope_end = []
-parameters = deque()
-prevtype = ""
+# parses one .sea file
+def parse_sea_file(filename):
+       scope_end = []
+       scope_end.append(32767)
+       parameters = deque()
+       prevtype = ""
 
-lines = args.infile.readlines(10000)
-while lines:
-       for line in lines:
-               (rowtype, remainder) = line.split(': ', 1)
-               if rowtype == 'Global':
-                       (rowfile, rowglob) = remainder.split(':', 1)
-                       (lineno, globalvar) = rowglob.split(' ', 1)
-                       lineno = int(lineno)
-                       vals = (rowfile, globalvar, lineno, 32767, 0)
-                       c.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
-               elif rowtype == 'Function':
-                       print 'nothing to do for entry type Function'
-               elif rowtype == 'Parameter':
-                       if prevtype == 'Function' or prevtype == 'Parameter':
-                               (rowfile, rowpar) = remainder.split(':', 1)
-                               (lineno, para) = rowpar.split(' ', 1)
+       try:
+               infile = open(filename, 'r')
+               print '======'
+               print 'reading file: ',filename
+       except IOError:
+               print '======'
+               print 'could not read file: ',filename
+               infile.close()
+               return
+
+       lines = infile.readlines(10000)
+       while lines:
+               for line in lines:
+                       (rowtype, remainder) = line.split(': ', 1)
+                       if rowtype == 'SystemGlobal':
+                               (rowfile, rowglob) = remainder.split(':', 1)
+                               (lineno, globalvar) = rowglob.split(' ', 1)
+                               vals = (filename, globalvar, 0, 32767, 0)
+                               sql_cur.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
+                       elif rowtype == 'Global':
+                               (rowfile, rowglob) = remainder.split(':', 1)
+                               (lineno, globalvar) = rowglob.split(' ', 1)
                                lineno = int(lineno)
-                               parameters.append((rowfile, para, lineno))
-               elif rowtype == 'Scope':
-                       (rowfile, scoperange) = remainder.split(':', 1)
-                       (scope_begin, scope_end_tmp) = scoperange.split(' ', 1)
-                       scope_begin = int(scope_begin)
-                       scope_end.append(int(scope_end_tmp) + 1)
-                       while prevtype == 'Parameter':
-                               try:
-                                       (rowfile, para, lineno) = 
parameters.popleft()
-                               except IndexError:
-                                       break
-                               vals = (rowfile, para, lineno, scope_end[-1], 0)
-                               c.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
-                       print 'entered new scope: ', scope_begin, '-', 
scope_end[-1]
-               elif rowtype == 'Expression':
-                       (rowfile, rowexpr) = remainder.split(':', 1)
-                       (lineno, iscall, expression) = rowexpr.split(' ', 2)
-                       lineno = int(lineno)
-                       iscall = int(iscall)
-                       if lineno > scope_end[-1]:
-                               scope_end.pop()
-                       vals = (rowfile, expression, lineno, scope_end[-1], 
iscall)
-                       c.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
-               else:
-                       print 'unknown entry type: ' + rowtype
-               prevtype = rowtype
-       conn.commit()
-       lines = args.infile.readlines(10000)
+                               vals = (rowfile, globalvar, lineno, 32767, 0)
+                               sql_cur.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
+                       elif rowtype == 'Function':
+                               print 'nothing to do for entry type Function'
+                       elif rowtype == 'Parameter':
+                               if prevtype == 'Function' or prevtype == 
'Parameter':
+                                       (rowfile, rowpar) = 
remainder.split(':', 1)
+                                       (lineno, para) = rowpar.split(' ', 1)
+                                       lineno = int(lineno)
+                                       parameters.append((rowfile, para, 
lineno))
+                       elif rowtype == 'Scope':
+                               (rowfile, scoperange) = remainder.split(':', 1)
+                               (scope_begin, scope_end_tmp) = 
scoperange.split(' ', 1)
+                               scope_begin = int(scope_begin)
+                               scope_end.append(int(scope_end_tmp) + 1)
+                               while prevtype == 'Parameter':
+                                       try:
+                                               (rowfile, para, lineno) = 
parameters.popleft()
+                                       except IndexError:
+                                               break
+                                       vals = (rowfile, para, lineno, 
scope_end[-1], 0)
+                                       sql_cur.execute('INSERT INTO Expression 
VALUES (NULL,?,?,?,?,?)', vals)
+                               print 'entered new scope: ', scope_begin, '-', 
scope_end[-1]
+                       elif rowtype == 'Expression':
+                               (rowfile, rowexpr) = remainder.split(':', 1)
+                               (lineno, iscall, expression) = rowexpr.split(' 
', 2)
+                               lineno = int(lineno)
+                               iscall = int(iscall)
+                               if lineno > scope_end[-1]:
+                                       scope_end.pop()
+                               vals = (rowfile, expression, lineno, 
scope_end[-1], iscall)
+                               sql_cur.execute('INSERT INTO Expression VALUES 
(NULL,?,?,?,?,?)', vals)
+                       else:
+                               print 'unknown entry type: ', rowtype
+                       prevtype = rowtype
+               sql_conn.commit()
+               lines = infile.readlines(10000)
 
 
+# traverses all .sea files in the input folder
+def traverse_sea_files(dummy, dirr, filess):
+       for child in filess:
+               if '.sea' == os.path.splitext(child)[1] and 
os.path.isfile(dirr+'/'+child):
+                       parse_sea_file(dirr+'/'+child)
+
+
+
+# parse it
+if os.path.isdir(args.INPUT):
+       os.path.walk(os.path.abspath(args.INPUT), traverse_sea_files, 0)
+elif os.path.isfile(args.INPUT):
+       parse_sea_file(args.INPUT)
+else:
+       print 'unknown INPUT, no rows written to ',outfilename
+
+# commit changes
+sql_conn.commit()
+
 # close db
-conn.close()
+sql_conn.close()
+
+
+