#!/usr/bin/python # Written by Reehan Shaikh # Last update: April 11, 2006 # Adapted from gloader.py import sys, os, signal, time from starter import Start import batch_ipcrm # set the program names VS_PROG = "uswitch" VM_PROG = "vmlinux" GR_PROG = "grouter" MCONSOLE_PROG = "uml_mconsole" SOCKET_NAME = "gini_socket" VS_PROG_BIN = VS_PROG VM_PROG_BIN = VM_PROG GR_PROG_BIN = GR_PROG MCONSOLE_PROG_BIN = MCONSOLE_PROG SRC_FILENAME = "gini_dist" # setup file name UML_WAIT_DELAY = 3 # wait delay between checking alive UML GROUTER_WAIT = 0.5 # wait delay between starting routers GINI_TMP_FILE = "gini_tmp_file" # tmp file used when checking alive UML LOG_FILE = "gdist_log" # log file for gloader messages SCREEN_LOG = False # telling to enable/disable screenlog file SSHOPTS = " -o StrictHostKeyChecking=false " # start the network def distGINI(myGINI, options, ips): "starting the GINI network components" # the starting order is important here # first switches, then routers, and at last UMLs. print "\nStarting GINI switches..." success = createVS(myGINI, options, ips) return success # create a switch for every UML and router interface so that these # components can interact with eaech other over sockets def createVS(myGINI, options, ips): "create the switch config file and start the switch for UML and router interfaces" success = True # create the switches for the UML interfaces for i in range(0, len(ips[2]), 2): for inters in ips[2][i + 1].interfaces: print "Starting Switch on machine %s for %s interface %s...\t" % (ips[2][i], ips[2][i + 1].name, inters.name) # find the specified directory of this specific IP for j in range(0, len(ips[0]), 2): if (ips[0][j] == ips[2][i]): break # create a SWITCH directory on remote machine, under the given specified directory from the IP file subSwitchDir = "%s/GINI/%s_Switch_%s" % (ips[0][j + 1], ips[2][i + 1].name, inters.name) os.system("ssh" + SSHOPTS + ips[2][i] + " rm -rf " + subSwitchDir) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[2][i] + " mkdir " + subSwitchDir) configFile = "%s.conf" % VS_PROG # create the config file configOut = open(configFile, "w") configOut.write("logfile %s.log\n" % VS_PROG) configOut.write("pidfile %s.pid\n" % VS_PROG) # enable port and remote configurations for h in range(0, len(ips[3]), 4): if (ips[3][h] == ips[2][i + 1].name and ips[3][h + 1] == inters.name): break configOut.write("port %d\n" % ips[3][h + 3]) targname = inters.target for m in range(0, len(ips[4]), 2): if (targname == ips[4][m + 1]): remoteIP = ips[4][m] break configOut.write("remote %s\n" % remoteIP) configOut.write("socket %s.ctl\n" % SOCKET_NAME) configOut.write("fork\n") configOut.close() ### ------- execute ---------- ### # create script files to run on remote machines scrpt = open("switch.sh", 'w') command = "cd %s/\n%s %s.conf" % (subSwitchDir, VS_PROG, VS_PROG) scrpt.write(command) scrpt.close() os.system("chmod 755 switch.sh") os.system("scp" + SSHOPTS + configFile + " switch.sh " + ips[2][i] + ":" + subSwitchDir + "/" + " >> ip_test_log") time.sleep(GROUTER_WAIT) command = " %s/switch.sh&" % subSwitchDir os.system("ssh" + SSHOPTS + ips[2][i] + command) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[2][i] + " rm -rf" + command) os.system("rm -rf switch.sh " + configFile) print "[OK]" # create the switches for the router interfaces for i in range(0, len(ips[1]), 2): for inters in ips[1][i + 1].netIF: print "Starting Switch on machine %s for %s interface %s...\t" % (ips[1][i], ips[1][i + 1].name, inters.name) for j in range(0, len(ips[0]), 2): if (ips[0][j] == ips[1][i]): break # create directory on remote machine subSwitchDir = "%s/GINI/%s_Switch_%s" % (ips[0][j + 1], ips[1][i + 1].name, inters.name) os.system("ssh" + SSHOPTS + ips[1][i] + " rm -rf " + subSwitchDir) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[1][i] + " mkdir " + subSwitchDir) configFile = "%s.conf" % VS_PROG # create the config file configOut = open(configFile, "w") configOut.write("logfile %s.log\n" % VS_PROG) configOut.write("pidfile %s.pid\n" % VS_PROG) # enable port and remote configurations for h in range(0, len(ips[3]), 4): if (ips[3][h] == ips[1][i + 1].name and ips[3][h + 1] == inters.name): break configOut.write("port %d\n" % ips[3][h + 3]) targname = inters.target for m in range(0, len(ips[4]), 2): if (targname == ips[4][m + 1]): remoteIP = ips[4][m] break configOut.write("remote %s\n" % remoteIP) configOut.write("socket %s.ctl\n" % SOCKET_NAME) configOut.write("fork\n") configOut.close() ### ------- execute ---------- ### # create script files to run on remote machines scrpt = open("switch.sh", 'w') command = "cd %s/\n%s %s.conf" % (subSwitchDir, VS_PROG, VS_PROG) scrpt.write(command) scrpt.close() os.system("chmod 755 switch.sh") os.system("scp" + SSHOPTS + configFile + " switch.sh " + ips[1][i] + ":" + subSwitchDir + "/" + " >> ip_test_log") time.sleep(GROUTER_WAIT) command = " %s/switch.sh&" % subSwitchDir os.system("ssh" + SSHOPTS + ips[1][i] + command) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[1][i] + " rm -rf" + command) os.system("rm -rf switch.sh " + configFile) print "[OK]" # now that the switches are started, start the routers print "\nStarting GINI Routers..." success = createVR(myGINI, options, ips) and success # now start the umls print "\nStarting GINI UMLs..." success = createVM(myGINI, options, ips) and success return success # distribute and start routers def createVR(myGINI, options, ips): "create router config file, and start the router" logOut = file(LOG_FILE, 'a') for i in range(0, len(ips[1]), 2): # find the specified directory on the remote machine for j in range(0, len(ips[0]), 2): if (ips[0][j] == ips[1][i]): break print "Starting Router %s on machine %s...\t" % (ips[1][i + 1].name, ips[1][i]) sys.stdout.flush() ### ------ config ---------- ### # create the router directory subRouterDir = "%s/GINI/%s" % (ips[0][j + 1], ips[1][i + 1].name) os.system("ssh" + SSHOPTS + ips[1][i] + " rm -rf " + subRouterDir) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[1][i] + " mkdir " + subRouterDir) configFile = "%s.conf" % GR_PROG # create the config file configOut = open(configFile, "w") for nwIf in ips[1][i + 1].netIF: socketName = "%s/GINI/%s_Switch_%s/%s.ctl" % (ips[0][j + 1], ips[1][i + 1].name, nwIf.name, SOCKET_NAME); configOut.write(getVRIFOutLine(nwIf, socketName)) configOut.close() ### ------- execute ---------- ### # go to the router directory to execute the command scrpt = open("router.sh", 'w') command = "cd %s/\n%s --config=%s.conf --interactive=1 %s" % (subRouterDir, GR_PROG, GR_PROG, ips[1][i + 1].name) scrpt.write(command) scrpt.close() os.system("chmod 755 router.sh") os.system("scp" + SSHOPTS + configFile + " router.sh " + ips[1][i] + ":" + subRouterDir + "/" + " >> ip_test_log") command = "screen -d -m " if (SCREEN_LOG): command += "-L " command += "-S %s ssh%s%s %s/router.sh" % (ips[1][i + 1].name, SSHOPTS, ips[1][i], subRouterDir) os.system(command) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[1][i] + " rm -rf " + subRouterDir + "/router.sh") os.system("rm -rf router.sh " + configFile) print "[OK]" logOut.close() return True # distribute and start the umls def createVM(myGINI, options, ips): "create UML config file, and start the UML" logOut = file(LOG_FILE, 'a') for i in range(0, len(ips[2]), 2): print "Starting UML %s on machine %s...\t" % (ips[2][i + 1].name, ips[2][i]) for j in range(0, len(ips[0]), 2): if (ips[0][j] == ips[2][i]): break # create the UML directory sys.stdout.flush() subUMLDir = "%s/GINI/%s" % (ips[0][j + 1], ips[2][i + 1].name) os.system("ssh" + SSHOPTS + ips[2][i] + " rm -rf " + subUMLDir) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[2][i] + " mkdir " + subUMLDir) # create command line command = createUMLCmdLine(ips[2][i + 1]) ### ---- process the UML interfaces ---- ### # it creates one config for each interface in the current directory # and returns a string to be attached to the UML exec command line for nwIf in ips[2][i + 1].interfaces: # name the socket, as per the specified switch socketName = "%s/GINI/%s_Switch_%s/%s.ctl" % (ips[0][j + 1], ips[2][i + 1].name, nwIf.name, SOCKET_NAME) configFile = "%s.sh" % nwIf.name # create the config file configOut = open(configFile, "w") configOut.write("ifconfig %s " % nwIf.name) configOut.write("%s\n" % nwIf.ip) for route in nwIf.routes: configOut.write("route add -%s %s " % (route.type, route.dest)) configOut.write("netmask %s " % route.netmask) configOut.write("gw %s\n" % route.gw) configOut.close() os.system("chmod 755 " + configFile) # prepare the output line outLine = "%s=daemon,%s,unix," % (nwIf.name, nwIf.mac) outLine += socketName command += "%s " % outLine scrpt = open("uml.sh", 'w') # because the uml looks for a MAC_ADRS.sh file, we move it using this script scrptcmd = "cd %s/\nmv %s /tmp/%s.sh" % (subUMLDir, configFile, nwIf.mac.upper()) scrpt.write(scrptcmd) scrpt.close() os.system("chmod 755 uml.sh") os.system("scp" + SSHOPTS + configFile + " uml.sh " + ips[2][i] + ":" + subUMLDir + "/" + " >> ip_test_log") time.sleep(GROUTER_WAIT) runscrpt = " %s/uml.sh&" % subUMLDir os.system("ssh" + SSHOPTS + ips[2][i] + runscrpt) time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + ips[2][i] + " rm -rf" + runscrpt) os.system("rm -rf uml.sh " + configFile) ### ------- execute ---------- ### # go to the UML directory to execute the command scrpt = open("startit.sh", 'w') cmd = "cd %s\n%s" % (subUMLDir, command) scrpt.write(cmd) scrpt.close() os.system("chmod 755 startit.sh") os.system("scp" + SSHOPTS + "startit.sh " + ips[2][i] + ":" + subUMLDir + "/" + " >> ip_test_log") time.sleep(GROUTER_WAIT) startUml = "screen -d -m -S %s ssh%s%s %s/startit.sh" % (ips[2][i + 1].name, SSHOPTS, ips[2][i], subUMLDir) os.system(startUml) time.sleep(UML_WAIT_DELAY) os.system("rm -rf startit.sh") print "[OK]" logOut.close() return True # taken from gloader def getVRIFOutLine(nwIf, socketName): "convert the router network interface specs into a string" outLine = "ifconfig add %s " % nwIf.name outLine += "-socket %s " % socketName outLine += "-addr %s " % nwIf.ip outLine += "-network %s " % nwIf.network outLine += "-hwaddr %s " % nwIf.nic if (nwIf.gw): outLine += "-gw %s " % nwIf.gw if (nwIf.mtu): outLine += "-mtu %s " % mwIf.mtu outLine += "\n" for route in nwIf.routes: outLine += "route add -dev %s " % nwIf.name outLine += "-net %s " % route.dest outLine += "-netmask %s " % route.netmask if (route.nexthop): outLine += "-gw %s" % route.nexthop outLine += "\n" return outLine # taken from gloader def createUMLCmdLine(uml): command = "" ## uml binary name if (uml.kernel): command += "%s " % uml.kernel else: command += "%s " % VM_PROG_BIN ## uml ID command += "umid=%s " % uml.name ## handle the file system option # construct the cow file name fileSystemName = getBaseName(uml.fileSystem.name) fsCOWName = "%s.cow" % fileSystemName if (uml.fileSystem.type.lower() == "cow"): command += "ubd0=%s,%s " % (fsCOWName, uml.fileSystem.name) else: command += "ubd0=%s " % uml.fileSystem.name ## handle the mem option if (uml.mem): command += "mem=%s " % uml.mem ## handle the boot option if (uml.boot): command += "con0=%s " % uml.boot return command # taken from gloader def getBaseName(pathName): "Extract the filename from the full path" pathParts = pathName.split("/") return pathParts[len(pathParts)-1] # stop the network def undistGINI(myGINI, options, ips): os.system("rm -rf ip_test_log") print "\nTerminating switches..." print "\nTerminating routers..." print "\nTerminating UMLs..." print "\nCleaning the interprocess message queues" batch_ipcrm.clean_ipc_queues() # since we are working on remote machines, we don't care about # the process there, so, we just ssh and kill -9 -1 (this kills # all the process linked to the user for i in range(0, len(ips[0]), 2): os.system("ssh" + SSHOPTS + " " + ips[0][i] + " kill -9 -1") if (not options.keepOld): print "\nDeleting GINI related files on remote machine %s...\n" % ips[0][i] command = " rm -rf %s/GINI" % ips[0][i + 1] os.system("ssh" + SSHOPTS + " " + ips[0][i] + command) return True # adapted from gloader with modifications def checkProcAlive(procName, ipdirs): alive = False # grep the GINI processes command = " ps aux | grep %s > %s" % (procName, GINI_TMP_FILE) for i in range(0, len(ipdirs), 2): os.system("ssh" + SSHOPTS + ipdirs[i] + command) # analyse the grepped output inFile = open(GINI_TMP_FILE) line = inFile.readline() while (line): if (line.find("grep") == -1): # don't consider the "grep" line userName = os.getlogin() lineParts = line.split() if (lineParts[0] == userName): # consider only the instances with the current user alive = True print "There is a live GINI %s on machine %s" % (procName, ipdirs[i]) line = inFile.readline() inFile.close() # clean up os.remove(GINI_TMP_FILE) return alive # adapted from gloader with modifications def writeSrcFile(options): "write the configuration in the setup file" outFile = open(SRC_FILENAME, "w") outFile.write("%s\n" % options.xmlFile) outFile.write("%s\n" % options.switchDir) outFile.write("%s\n" % options.routerDir) outFile.write("%s\n" % options.umlDir) outFile.write("%s\n" % options.binDir) outFile.write("%s\n" % options.ipSpecs) outFile.close() # taken from gloader def deleteSrcFile(): "delete the setup file" if (os.access(SRC_FILENAME, os.W_OK)): os.remove(SRC_FILENAME) else: print "Could not delete the GINI setup file" # adapted from gloader with modifications def checkAliveGini(ips): "check any of the gini components already running" # Modified to check every machine in our ipSpecs file result = False if checkProcAlive(VS_PROG_BIN, ips[0]): result = True if checkProcAlive(VM_PROG_BIN, ips[0]): result = True if checkProcAlive(GR_PROG_BIN, ips[0]): result = True return result #### -------------- MAIN start ----------------#### # create the program processor. This # 1. accepts and process the command line options # 2. creates XML processing engine, that in turn # a) validates the XML file # b) extracts the DOM object tree # c) populates the GINI network class library # d) performs some semantic/syntax checkings on # the extracted specification # e) validates the IP file for distribution myProg = Start(sys.argv[0], SRC_FILENAME) if (not myProg.processOptions(sys.argv[1:])): sys.exit(1) options = myProg.options # Get the valid IPs and directories from the ipSpecs file # Also check if the IPs and directories are valid # we validate by: scp a file into given machine and directory # ssh and remove the file. Once this is validated, we don't # have to error-check these operations anymore ipfilehandle = open(options.ipSpecs, 'r') lines = ipfilehandle.readlines() iptest = open("ip_test_log", 'w') iptest.close() ginitest = open("gini_ip_test", 'w') ginitest.write("This is a test file\nIt should not be here\nIt should have been deleted automatically\nDelete it if you can read this!!!") ginitest.close() ipdircombos = [] res = False for line in lines: a = line.split("\n") b = a[0].split(":") ipdircombos.append(b[0]) ipdircombos.append(b[1]) if ((not myProg.undistOpt) or (not options.keepOld)): os.system("ssh" + SSHOPTS + b[0] + " rm -rf " + b[1] + "/GINI") time.sleep(GROUTER_WAIT) os.system("ssh" + SSHOPTS + b[0] + " mkdir " + b[1] + "/GINI") i = os.system("scp" + SSHOPTS + "gini_ip_test " + a[0] + "/GINI/ >> ip_test_log") if (not i == 0): print "Problem with machine or directory %s" % a[0] res = True if (i == 0): os.system("ssh" + SSHOPTS + b[0] + " rm -rf " + b[1] + "/GINI/gini_ip_test >> ip_test_log") print "Machine and directory valid on %s" % a[0] os.system("rm -rf gini_ip_test") ipfilehandle.close() if (res): sys.exit(1) # get the populated GINI network class # its structure is the same as the XML specification myGINI = myProg.giniNW # We don't distribute switches if (len(myGINI.switches) > 0): print "\nCannot distriute switches...sorry" print "These cannot be in the topology" sys.exit(1) # Let the user know about the number of IPs total_ips_req = len(myGINI.vr) + len(myGINI.vm) total_ips_giv = len(ipdircombos) / 2 if (total_ips_req > total_ips_giv): print "\nThe given IPs aren't enough" print "There will be more than one GINI component on some machines\n" # Calculate which component will map to which IP ipvrcombos = [] ipvmcombos = [] ipcompcombos = [] j = 0 for i in range(len(myGINI.vr)): ipvrcombos.append(ipdircombos[j]) ipvrcombos.append(myGINI.vr[i]) ipcompcombos.append(ipdircombos[j]) ipcompcombos.append(myGINI.vr[i].name) j = (j + 2) % len(ipdircombos) for i in range(len(myGINI.vm)): ipvmcombos.append(ipdircombos[j]) ipvmcombos.append(myGINI.vm[i]) ipcompcombos.append(ipdircombos[j]) ipcompcombos.append(myGINI.vm[i].name) j = (j + 2) % len(ipdircombos) # Calculate switch port properties. If there is a # link in the GINI topology between two components, # then the switches for these components must have # the same port number and their respective remote # addresses should refer to each other ipports = [] for i in myGINI.vm: for j in i.interfaces: ipports.append(i.name) ipports.append(j.name) ipports.append(j.target) ipports.append(0) for i in myGINI.vr: for j in i.netIF: ipports.append(i.name) ipports.append(j.name) ipports.append(j.target) ipports.append(0) # find available ports such that they match on both remote machines # switches are initialized with port and remote ip fields but the # host and remote ip are only talking to the same port (ie. if the # host machine has a switch on port x then the remote machine must # have a listening switch on port x as well if (not myProg.undistOpt): switchport = 1115 for i in range(3, len(ipports), 4): if ipports[i] == 0: j = 1 while (not ipports[i - 3] == ipcompcombos[j]): j += 2 k = 1 while (not ipports[i - 1] == ipcompcombos[k]): k += 2 os.system("ssh" + SSHOPTS + ipcompcombos[j - 1] + " netstat > m1ports") os.system("ssh" + SSHOPTS + ipcompcombos[k - 1] + " netstat > m2ports") check = True while (check): command = "grep -w %d m1ports >> ip_test_log" % switchport m1 = os.WEXITSTATUS(os.system(command)) command = "grep -w %d m2ports >> ip_test_log" % switchport m2 = os.WEXITSTATUS(os.system(command)) if (m1 == 1 and m2 == 1): ipports[i] = switchport for x in range(3, len(ipports), 4): if (ipports[i - 3] == ipports[x - 1] and ipports[i - 1] == ipports[x - 3]): break ipports[x] = switchport switchport += 1 check = False else: switchport += 1 os.system("rm -rf m1ports m2ports") # Store the mappings so we can pass them around ips = [] # Zeroth element is IP and Directory tuples ips.append(ipdircombos) # First element is IP and Router tuples ips.append(ipvrcombos) # Second element is IP and UML tuples ips.append(ipvmcombos) # Third element is Switch port configurations ips.append(ipports) # Forth element is IP and component tuples # UMLs and Router in one list ips.append(ipcompcombos) # reset the log file if (os.access(LOG_FILE, os.F_OK)): os.remove(LOG_FILE) # distribute or undistribute GINI network print "" if (myProg.undistOpt): # terminate the current distributed specification print "Terminating GINI network..." success = undistGINI(myGINI, options, ips) if (success): print "\nGINI network is undistributed and terminated!!\n" else: print "\nThere are errors in GINI network termination" print "Check the logfile %s for more details" % LOG_FILE print "You might have to terminate the orphaned processes manually\n" sys.exit(1) else: # create a distributed GINI instance if (not options.keepOld): # fail if a GINI already alive if checkAliveGini(ips): sys.exit(1) # create network with current specifcation print "Creating and distributing a GINI network..." success = distGINI(myGINI, options, ips) writeSrcFile(options) if (success): print "\nGINI network up, running and distributed!!\n" else: print "\nProblem in creating GINI network" print "Check the log file %s for details" % LOG_FILE print "** Run gdist -y to terminate the partially started ", print "GINI instance before starting another one **\n" sys.exit(1) sys.exit(0)