[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r25744 - in gnunet-planetlab/gplmt: . contrib gplmt scripts
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r25744 - in gnunet-planetlab/gplmt: . contrib gplmt scripts |
|
Date: |
Thu, 10 Jan 2013 14:17:19 +0100 |
Author: wachs
Date: 2013-01-10 14:17:19 +0100 (Thu, 10 Jan 2013)
New Revision: 25744
Modified:
gnunet-planetlab/gplmt/contrib/master_template.cfg
gnunet-planetlab/gplmt/contrib/test.conf
gnunet-planetlab/gplmt/gplmt.py
gnunet-planetlab/gplmt/gplmt/Configuration.py
gnunet-planetlab/gplmt/scripts/gplmt-api.py
Log:
changes to buildbot configuration generation
Modified: gnunet-planetlab/gplmt/contrib/master_template.cfg
===================================================================
--- gnunet-planetlab/gplmt/contrib/master_template.cfg 2013-01-10 12:36:06 UTC
(rev 25743)
+++ gnunet-planetlab/gplmt/contrib/master_template.cfg 2013-01-10 13:17:19 UTC
(rev 25744)
@@ -19,7 +19,7 @@
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = 9989
+c['slavePortnum'] = [%GPLMT_SLAVES_PORT]
####### CHANGESOURCES
@@ -109,7 +109,7 @@
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
- auth=auth.BasicAuth([("%GPLMT_FORCE_USER","%GPLMT_FORCE_PWD")]),
+ %GPLMT_FORCE_ENABLE
auth=auth.BasicAuth([("%GPLMT_FORCE_USER","%GPLMT_FORCE_PWD")]),
gracefulShutdown = False,
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = 'auth',
@@ -118,7 +118,7 @@
stopAllBuilds = 'auth',
cancelPendingBuild = 'auth',
)
-c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+c['status'].append(html.WebStatus(http_port=%GPLMT_WEB_PORT, authz=authz_cfg))
####### PROJECT IDENTITY
@@ -135,7 +135,7 @@
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
-c['buildbotURL'] = "http://localhost:8010/";
+c['buildbotURL'] = "http://localhost:%GPLMT_WEB_PORT/";
####### DB URL
Modified: gnunet-planetlab/gplmt/contrib/test.conf
===================================================================
--- gnunet-planetlab/gplmt/contrib/test.conf 2013-01-10 12:36:06 UTC (rev
25743)
+++ gnunet-planetlab/gplmt/contrib/test.conf 2013-01-10 13:17:19 UTC (rev
25744)
@@ -1,31 +1,25 @@
[gplmt]
-nodes = contrib/localhost.nodes
-nodes = contrib/test_node.nodes
-
-#tasks = contrib/tasks.xml
-tasks = contrib/simpletasks.xml
-#tasks = contrib/sftp_tasks.xml
-
+#nodes = contrib/test_node.nodes
# Which notification mechanism to use:
# simple: print messages to stdout
-notification = simple
+#notification = simple
+notification = result
+tasks = contrib/tasklists/check_node.xml
[planetlab]
-#slice = mwachs
-slice = tumple_gnunet_deployment
-
+slice = tumple_gnunet
# Configuration for Planetlab API
# Use PlanetLab nodes instead of nodes file
-use_pl_nodes = no
+use_pl_nodes = yes
# PlanetLab Europe
api_url = https://www.planet-lab.eu/PLCAPI/
# Planetlab
#api_url = https://www.planet-lab.org/PLCAPI/
# Planetlab username
-username = address@hidden
+username = <pl_username>
# Planetlab password
-password = <password>
+password = <pl_pw>
[ssh]
# Order of ssh authentication:
@@ -34,11 +28,28 @@
# Any key we can find through an SSH agent
# Any "id_rsa" or "id_dsa" key discoverable in ~/.ssh/
# Plain password auth, if a ssh_password was given
-
#ssh_keyfile = <keyfile>
ssh_password = <password>
-
+# protocoll for put get operations (Planetlab does not support SFTP)
+ssh_transfer = scp
+#ssh_transfer = sftp
# Use system's SSH "known hosts" file
ssh_use_known_hosts = yes
# Add node hostkeys automatically
-add_unkown_hostkeys = yes
\ No newline at end of file
+add_unkown_hostkeys = yes
+
+[buildbot]
+# configuration template
+template = ../../gplmt/contrib/master_template.cfg
+result_task_file = result.tasks
+result_cfg = result.cfg
+buildbot_command = "buildslave create-slave ./bot "
+master = gnunet.org:9990
+slaveport = 9990
+webport = 8011
+# set to "yes" to allow force building
+force_enable = no
+force_user = <force_user>
+force_pw = <force_pw>
+
+
Modified: gnunet-planetlab/gplmt/gplmt/Configuration.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt/Configuration.py 2013-01-10 12:36:06 UTC
(rev 25743)
+++ gnunet-planetlab/gplmt/gplmt/Configuration.py 2013-01-10 13:17:19 UTC
(rev 25744)
@@ -57,6 +57,13 @@
self.bb_template = ""
self.bb_task_file = ""
self.bb_result_cfg = ""
+ self.bb_bb_cmd = ""
+ self.bb_master = ""
+ self.bb_force_enable = ""
+ self.bb_force_user = ""
+ self.bb_force_pw = ""
+ self.bb_webport = 0
+ self.bb_slaveport = 0
def load (self):
if (None == self.filename):
default_cfg = os.path.expanduser("~") + os.sep + ".gplmt" +
os.sep+ "gplmt.conf"
@@ -100,7 +107,7 @@
try:
self.pl_password = config.get("planetlab", "password")
except ConfigParser.NoOptionError as e:
- pass
+ pass
except ConfigParser.NoSectionError:
pass
@@ -130,13 +137,37 @@
except ConfigParser.NoOptionError as e:
pass
try:
- self.bb_result_task_file = config.get("buildbot",
"result_task_file")
+ self.bb_task_file = config.get("buildbot", "result_task_file")
except ConfigParser.NoOptionError as e:
pass
try:
self.bb_result_cfg = config.get("buildbot", "result_cfg")
except ConfigParser.NoOptionError as e:
- pass
+ pass
+ try:
+ self.bb_bb_cmd = config.get("buildbot", "buildbot_command")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.bb_master = config.get("buildbot", "master")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.bb_force_pw = config.get("buildbot", "force_pw")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.bb_force_enable = config.get("buildbot", "force_enable")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.bb_slaveport = config.get("buildbot", "slaveport")
+ except ConfigParser.NoOptionError as e:
+ pass
+ try:
+ self.bb_webport = config.get("buildbot", "webport")
+ except ConfigParser.NoOptionError as e:
+ pass
except ConfigParser.NoSectionError:
pass
Modified: gnunet-planetlab/gplmt/gplmt.py
===================================================================
--- gnunet-planetlab/gplmt/gplmt.py 2013-01-10 12:36:06 UTC (rev 25743)
+++ gnunet-planetlab/gplmt/gplmt.py 2013-01-10 13:17:19 UTC (rev 25744)
@@ -1,4 +1,4 @@
-#!env python
+#!/usr/bin/env python
#
# This file is part of GNUnet.
# (C) 2010 Christian Grothoff (and other contributing authors)
Modified: gnunet-planetlab/gplmt/scripts/gplmt-api.py
===================================================================
--- gnunet-planetlab/gplmt/scripts/gplmt-api.py 2013-01-10 12:36:06 UTC (rev
25743)
+++ gnunet-planetlab/gplmt/scripts/gplmt-api.py 2013-01-10 13:17:19 UTC (rev
25744)
@@ -1,5 +1,5 @@
-#!env python
-import sys, os, xmlrpclib, getopt, getpass
+#!/usr/bin/env python
+import sys, os, xmlrpclib, getopt, getpass, random
def usage():
@@ -11,7 +11,14 @@
-s, --slice= Planetlab slice\n\
-n, --nodes= Files containing nodes\n\
-h, --help print this help\n\
- -o, --operation= all,my,add_nodes_file, add_nodes_all, del_nodes_file\n\
+ -o, --operation= all,my,add_nodes_file, add_nodes_all, del_nodes_file,
create_buildbot_cfg\n\
+ \n\
+ Parameters for buildbot configuration creation\n\
+ -t, --template= buildbot configuration template\n\
+ -b, --bconfig filename for resulting buildbot configuration\n\
+ -r, --btask filename for resulting tasklist\n\
+ -w, --bcmd buidlbot command to execute\n\
+ Additional paramaters need to be configured in configuration file\n\
Report bugs to address@hidden \n\
GNUnet home page: http://www.gnu.org/software/gnunet/ \n\
General help using GNU software: http://www.gnu.org/gethelp/";
@@ -23,6 +30,10 @@
cfgfile = None
nodesfile = None
op = None
+bconfig = None
+template = None
+bb_task_file = None
+bb_cmd = None
def parse_arg ():
global cfgfile
@@ -31,8 +42,12 @@
global pl_slicename
global op
global nodesfile
+ global bconfig
+ global template
+ global bb_task_file
+ global bb_cmd
try:
- opts, args = getopt.getopt(sys.argv[1:], "hc:u:p:s:o:n:", ["help",
"config", "pl_user=", "pl_password=", "slice=", "operation=", "nodes="])
+ opts, args = getopt.getopt(sys.argv[1:], "w:r:b:hc:u:p:s:o:n:t:",
["bcmd=","btask=", "bconfig=","help", "config", "pl_user=", "pl_password=",
"slice=", "operation=", "nodes=", "template"])
except getopt.GetoptError, err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
@@ -51,7 +66,17 @@
elif o in ("-s", "--slice"):
pl_slicename = a
elif o in ("-n", "--nodes"):
- nodesfile = a
+ nodesfile = a
+ elif o in ("-t", "--template"):
+ template = a
+ elif o in ("-b", "--bconfig"):
+ bconfig = a
+ elif o in ("-r", "--btask"):
+ bb_task_file = a
+ elif o in ("-w", "--bcmd"):
+ bb_cmd = a
+
+
elif o in ("-o", "--operation"):
if (a == "all"):
op = "all"
@@ -62,7 +87,9 @@
elif (a == "add_nodes_all"):
op = "add_nodes_all"
elif (a == "del_nodes_file"):
- op = "del_nodes_file"
+ op = "del_nodes_file"
+ elif (a == "create_buildbot_cfg"):
+ op = "create_buildbot_cfg"
else:
usage()
sys.exit()
@@ -70,6 +97,16 @@
assert False, "unhandled option"
def update_conf (configuration):
+ global cfgfile
+ global pl_user
+ global pl_password
+ global pl_slicename
+ global op
+ global nodesfile
+ global bconfig
+ global template
+ global bb_task_file
+ global bb_cmd
if (None != pl_user):
configuration.pl_user = pl_user
if (None != pl_password):
@@ -77,7 +114,15 @@
if (None != pl_slicename):
configuration.pl_slicename = pl_slicename
if (None != pl_password):
- configuration.pl_password = pl_password
+ configuration.pl_password = pl_password
+ if (None != template):
+ configuration.bb_template = template
+ if (None != bconfig):
+ configuration.bb_result_cfg = bconfig
+ if (None != bb_task_file):
+ configuration.bb_task_file = bb_task_file
+ if (None != bb_cmd):
+ configuration.bb_bb_cmd = bb_cmd
def load_nodes (filename):
if (False == os.path.exists(filename)):
@@ -230,7 +275,141 @@
except Exception as e:
print "Failed to delete node :" + str(e)+ " from slice " +
configuration.pl_slicename
-
+def create_buildbot_cfg (configuration):
+ global nodesfile
+ server = xmlrpclib.ServerProxy(configuration.pl_api_url)
+ # PlanetLab auth struct
+ auth = {}
+ auth['Username'] = configuration.pl_username
+ auth['AuthString'] = configuration.pl_password
+ auth['AuthMethod'] = "password"
+ # PlanetLab Slice data
+ slice_data = {}
+ slice_data['name'] = configuration.pl_slicename
+
+ # Load nodes
+ node_hostnames = []
+ if (("" != configuration.pl_username) and
+ ("" != configuration.pl_password) and
+ ("" != configuration.pl_slicename) and
+ (None == nodesfile)):
+ # Request nodes assigned to slice
+ try:
+ node_ids = server.GetSlices(auth, [slice_data['name']],
['node_ids'])[0]['node_ids']
+ node_hostnames = [node['hostname'] for node in
server.GetNodes(auth, node_ids, ['hostname'])]
+ except Exception as e:
+ print "Error while retrieving node list: " + str(e)
+ sys.exit (2)
+ elif (None != nodesfile):
+ # Use nodes files
+ try:
+ f = open (nodesfile, 'r')
+ except IOError as e:
+ print "Could not open node file: " + str(e)
+ sys.exit (2)
+ for line in f:
+ node_hostnames.append(line)
+ f.close()
+ else:
+ print "No node file or PlanetLab credentials given"
+ sys.exit (2)
+
+ if ("" == configuration.bb_template):
+ print "No configuration template given"
+ sys.exit (2)
+ if ("" == configuration.bb_task_file):
+ print "No file to store resulting tasks given"
+ sys.exit (2)
+ if ("" == configuration.bb_result_cfg):
+ print "No file to store resulting configuration given"
+ sys.exit (2)
+ if ("" == configuration.bb_bb_cmd):
+ print "No buildbot command given"
+ sys.exit (2)
+ if ("" == configuration.bb_master):
+ print "No buildbot master given"
+ sys.exit (2)
+ if (0 == configuration.bb_slaveport):
+ print "No buildbot slave port given"
+ sys.exit (2)
+ if (0 == configuration.bb_webport):
+ print "No buildbot web port given"
+ sys.exit (2)
+
+
+ # Create buildslave information
+ template = {}
+ master_cfg_slaves = ""
+ master_cfg_builder_definition = ""
+ master_cfg_builder_summary = ""
+ master_cfg_scheduler_builders = ""
+ slave_cmds = ""
+ c = 0
+ try:
+ f_cmd = open(configuration.bb_task_file, 'w')
+ except IOError as e:
+ print "Cannot open output command file "+ configuration.bb_task_file
+": "+str(e)
+ sys.exit (2)
+
+ for node in node_hostnames:
+ node = node.strip()
+ print "Found node '" + node + "'"
+ password =
''.join([random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890')
for i in range(64)])
+ # slaves
+ master_cfg_slaves += 'BuildSlave("'+node+'","'+password+'"),\n'
+ # builder definition
+ master_cfg_builder_definition += "builder"+str(c)+" = {'name': \"" +
node +"\"," + "'slavename': \"" + node + "\",'builddir':\"" + node
+"\",'factory': f,'category': \"GNUnet\",}\n"
+ # scheduler
+ master_cfg_scheduler_builders += '"'+node+'", '
+ # builder summary
+ master_cfg_builder_summary += "builder"+str(c)+", "
+ # cmd
+ f_cmd.writelines (node +";" + configuration.bb_bb_cmd + " " +
configuration.bb_master + " " + node + " "+ password + "\n")
+ c += 1
+ try:
+ f_cmd.close()
+ except IOError as e:
+ print "Cannot close output task file "+ configuration.bb_task_file +":
"+str(e)
+ sys.exit (2)
+
+ #print master_cfg_slaves
+ #print master_cfg_builder_definition
+ #print master_cfg_builder_summary
+
+ # Create master.cfg
+ master_file = ""
+ try:
+ f_tmpl = open(configuration.bb_template, 'r')
+ except IOError as e:
+ print "Cannot open template file "+ configuration.bb_template
+ sys.exit (2)
+
+ for line in f_tmpl:
+ line.strip()
+ line = line.replace ("%GPLMT_BUILDER_DEFINITION",
master_cfg_builder_definition)
+ line = line.replace ("%GPLMT_BUILDER_SUMMARY",
master_cfg_builder_summary)
+ line = line.replace ("%GPLMT_SLAVES", master_cfg_slaves)
+ line = line.replace ("%GPLMT_SCHEDULER_BUILDERS",
master_cfg_scheduler_builders)
+ line = line.replace ("%GPLMT_WEB_PORT", configuration.bb_webport)
+ line = line.replace ("%GPLMT_SLAVES_PORT", configuration.bb_slaveport)
+ if ("" != configuration.bb_force_user):
+ line = line.replace ("%GPLMT_FORCE_USER",
configuration.bb_force_user)
+ if ("" != configuration.bb_force_pw):
+ line = line.replace ("%GPLMT_FORCE_PWD",
configuration.bb_force_pw)
+ if ("yes" == configuration.bb_force_enable):
+ line = line.replace ("%GPLMT_FORCE_ENABLE", "")
+ else:
+ line = line.replace ("%GPLMT_FORCE_ENABLE", "#")
+ master_file += line
+ f_tmpl.close()
+ try:
+ f_cfg = open(configuration.bb_result_cfg, 'w')
+ except IOError as e:
+ print "Cannot open template file "+ configuration.bb_result_cfg +":
"+str(e)
+ sys.exit (2)
+ f_cfg.writelines(master_file)
+ f_cfg.close()
+
def main():
global nodesfile
global cfgfile
@@ -260,7 +439,7 @@
if (None != nodesfile):
nodes = load_nodes(nodesfile)
if (None == nodes):
- print "Nodes file " +nodesfile+ "not found"
+ print "Nodes file " +nodesfile+ " not found"
sys.exit(1)
else:
nodes = None
@@ -276,7 +455,9 @@
elif ((None != nodes) and (op == "del_nodes_file")):
del_nodes (configuration, nodes)
elif (op == "add_nodes_all"):
- add_to_all_nodes(configuration)
+ add_to_all_nodes(configuration)
+ elif (op == "create_buildbot_cfg"):
+ create_buildbot_cfg (configuration)
else:
usage()
sys.exit()
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r25744 - in gnunet-planetlab/gplmt: . contrib gplmt scripts,
gnunet <=