[cvs] / gkb / gkb.py  

cvs: gkb/gkb.py

Diff for /gkb/gkb.py between version 1.1.1.1 and 1.11

version 1.1.1.1, Wed Jul 21 17:32:57 2004 UTC version 1.11, Sat Aug 14 06:42:14 2004 UTC
Line 1 
Line 1 
 #!/usr/bin/python  #!/usr/bin/python
 #  #
 # GKB - GNU Kernel Builder  # GKB - GNU Kernel Builder
 # Copyright (C) 2003-2004 Mark Guertin, Tobias McNulty  # Copyright (C) 2003-2004 Tobias McNulty, Mark Guertin
 #  #
 # This program is free software; you can redistribute it and/or modify  # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by  # it under the terms of the GNU General Public License as published by
Line 30 
Line 30 
 from ClientForm import ParseResponse  from ClientForm import ParseResponse
   
 debug=0  debug=0
 clean=1  clean=0
   dont_build=1
 verbose=1  verbose=1
   
 # parse host-specific configuration information from gkb.cfg  # parse host-specific configuration information from gkb.cfg
Line 61 
Line 62 
 mastertrees={}  mastertrees={}
 builds={}  builds={}
   
   class Error(Exception):
           """Base class for exceptions in this module."""
           pass
   
   class BuildError(Error):
           """Exception raised for build-level errors.
   
           Attributes:
                   message -- explanation of the error
           """
   
           def __init__(self, message):
                   self.message = message
   
   class FatalError(Error):
           """Exception for fatal errors that prevent program continuation.
   
           Attributes:
                   message -- explanation of why the specific transition is not allowed
           """
   
           def __init__(self, previous, next, message):
                   self.message = message
   
 def printverbose(myoutput):  def printverbose(myoutput):
         """Supporting method to output info to stdout if verbosity level is set"""          """Supporting method to output info to stdout if verbosity level is set"""
         if verbose==1:          if verbose==1:
Line 72 
Line 97 
         else:          else:
                 printverbose("gkb : " + text)                  printverbose("gkb : " + text)
   
 def deathbyerror(mymsg):  
         """Supporting method to print error to stdout and return system error -1 on exit"""  
         print mymsg  
         sys.exit(-1)  
   
 def verifydir(mydir,kinfo=0):  def verifydir(mydir,kinfo=0):
         """Supporting method to verify a given  exists, if not it will create it"""          """Supporting method to verify a given  exists, if not it will create it"""
         log("verifying directory " + mydir,kinfo)          log("verifying directory " + mydir,kinfo)
Line 93 
Line 113 
 def verifyfile(myfile,kinfo):  def verifyfile(myfile,kinfo):
         """Supporting method to verify file exists, if not it will exit with error"""          """Supporting method to verify file exists, if not it will exit with error"""
         if not os.path.isfile(myfile):          if not os.path.isfile(myfile):
                 deathbyerror("%s : cannot find file expected at %s, exiting" % (kinfo["name"],myfile))                  raise BuildError("cannot find file expected at %s, exiting" % myfile)
         return myfile          return myfile
   
 def md5sum(fileobj):  def md5sum(fileobj):
Line 107 
Line 127 
         digest = md.hexdigest()          digest = md.hexdigest()
         return digest          return digest
   
 #def krn_ftpput(filename):  
 #       """uploads the file fileobj to the distribution site"""  
         #ftp = FTP(mftphost)  
         #er = ftp.login(mftpuser,mftppass)  
         #er = ftp.storbinary("STOR "+os.path.basename(filename),open(filename))  
         #er = ftp.quit()  
   
 def krn_querymgr(command,kinfo):  def krn_querymgr(command,kinfo):
         """queries the build host manager with a variety of commands, such as checkout, checkin, etc."""          """queries the build host manager with a variety of commands, such as checkout, checkin, etc."""
         version=krn_localversion(kinfo)          version=krn_localversion(kinfo)
Line 139 
Line 152 
         log("calling gkb_getsource",kinfo)          log("calling gkb_getsource",kinfo)
         gkb_getsource(kinfo)          gkb_getsource(kinfo)
   
         # now we'll go into the work          # go into the work directory
           chdir(workdir)
   
           # archive the clean source for later uploading
           os.spawnlp(os.P_WAIT, "tar xjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"])
   
           # now we'll go into the tree's work dir
         chdir(kinfo["workdir"],kinfo)          chdir(kinfo["workdir"],kinfo)
   
         # check for patches and apply if necessary          # check for patches and apply if necessary
Line 162 
Line 181 
         """build a kernel, version 2.4.x"""          """build a kernel, version 2.4.x"""
   
         if krn_querymgr("checkout",kinfo):          if krn_querymgr("checkout",kinfo):
                   try:
                 myversion=krn_localversion(kinfo)                  myversion=krn_localversion(kinfo)
   
                 # fetch and cp the config file to work/.config                  # fetch and cp the config file to work/.config
Line 169 
Line 189 
   
                 # **Note** : we set the preprocessing command to premake inline instead                  # **Note** : we set the preprocessing command to premake inline instead
                 # of globally as it is only needed in this target                  # of globally as it is only needed in this target
                           if dont_build==0:
                 gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts)                  gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts)
   
                 # give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature                  # give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature
                 if clean==1:                          if (clean==1 and dont_build==0):
                         gkb_runmake("clean", kinfo, premake, makeopts)                          gkb_runmake("clean", kinfo, premake, makeopts)
   
                           if dont_build==0:
                 gkb_runmake("dep", kinfo, premake, makeopts)                  gkb_runmake("dep", kinfo, premake, makeopts)
   
                 gkb_runmake(kinfo["binname"], kinfo, premake, makeopts)                  gkb_runmake(kinfo["binname"], kinfo, premake, makeopts)
   
                 # We should check to see if binary built ok, if not bail out                  # We should check to see if binary built ok, if not bail out
Line 190 
Line 211 
                         copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)                          copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)
                 else:                  else:
                         # the binary is not there, inform user and bail out with error                          # the binary is not there, inform user and bail out with error
                         deathbyerror("%s : %s is not present, assuming build failure and exiting.  See log for details." % (kinfo["name"], kbinloc))                                  raise BuildError("%s is not present, assuming build failure and exiting.  See log for details." % kbinloc)
   
                 # now that we know he binary built, let's continue                  # now that we know he binary built, let's continue
                           if dont_build==0:
                 gkb_runmake("modules",kinfo, premake, makeopts)                  gkb_runmake("modules",kinfo, premake, makeopts)
   
                 # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead                  # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead
                 # of globally as it is only needed in this target                  # of globally as it is only needed in this target
                 gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))                  gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))
   
                           # compress and upload source archive
                           chdir(kinfo["workdir"]+"/..",kinfo)
                           archive_name = "src-%s.tar.bz2" % kinfo["mastertree"],
   
                           # compress and upload kernel binary
                 chdir(bindir,kinfo)                  chdir(bindir,kinfo)
                 log("compressing binary archive %s-%s" % (kinfo["name"],myversion),kinfo)                          archive_name = "linux-%s-%s.tar.bz2" % (kinfo["name"], myversion)
                 if os.system("tar cjf linux-%s-%s.tar.bz2 %s" % (kinfo["name"], myversion, os.path.basename(mybindir))):  
                         deathbyerror("%s : failed to `tar cjf` %s-%s" % (kinfo["name"], kinfo["name"], myversion))  
   
                 os.system("rm -rf %s" % mybindir)                          log("compressing binary archive "+archive_name,kinfo)
   
                           if os.spawnlp(os.P_WAIT, "tar cjf "+archive_name+" "+os.path.basename(mybindir)):
                                   raise BuildError("failed to `tar cjf %s`" %  archive_name)
   
                           os.spawnlp(os.P_WAIT, "rm -rf %s" % mybindir)
   
                           krn_upload(archive_name,"kernel",myversion,kinfo)
                 krn_querymgr("checkin",kinfo)                  krn_querymgr("checkin",kinfo)
   
                           if (os.fork() == 0):
                                   #in child
                                   krn_upload(workdir + "/" + kinfo["mastertree"] + ".tar.bz2", "source", myversion, kinfo)
   
                   except BuildError, e:
                           log(e.message, kinfo)
                           krn_querymgr("checkin",kinfo)
                   except:
                           krn_querymgr("checkin",kinfo)
                           raise
   
 def krn_build26(kinfo):  def krn_build26(kinfo):
         """build a kernel, version 2.6"""          """build a kernel, version 2.6"""
   
Line 236 
Line 278 
         syncline=args+" "+kinfo["workdir"]          syncline=args+" "+kinfo["workdir"]
   
         log("running rsync : %s %s" % (syncoptions, syncline),kinfo)          log("running rsync : %s %s" % (syncoptions, syncline),kinfo)
         if os.system("%s %s > %s/%s/rsync.log 2>&1" % (syncoptions, syncline, logdir, kinfo["name"])):          if os.spawnlp(os.P_WAIT, "%s %s > %s/%s/rsync.log 2>&1" % (syncoptions, syncline, logdir, kinfo["name"])):
                 deathbyerror("%s : sync failed, tried %s.  See log for details." % (kinfo["name"], syncline) )                  raise BuildError("sync failed, tried %s.  See log for details." %  syncline)
   
         # Gerk comment:  
         # This above might cause problems ... it (I think) relies on anything coming from stderr to tell  
         # if it has in fact died... we should test this out.  If this is the case me might need to make  
         # a stub of some sorts to handle the build/error return process...  
   
   
 def get_wget(kinfo):  def get_wget(kinfo):
         #needs work          #needs work
Line 253 
Line 289 
         log("fetching source archive " % args,kinfo)          log("fetching source archive " % args,kinfo)
         mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])          mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])
   
         if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):          if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
                 deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"]))                  raise BuildError("unable to download configfile, aborting.")
   
         log("decompressing source file",kinfo)          log("decompressing source file",kinfo)
         if os.system("tar xjf " % (myconfigfile,msite,kinfo["name"])):          if os.spawnlp(os.P_WAIT, "tar xjf " % (myconfigfile,msite,kinfo["name"])):
                 deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"]))                  raise BuildError("unable to decompress source file, aborting.")
   
 def get_vanilla(kinfo):  def get_vanilla(kinfo):
         args=mastertrees[kinfo["mastertree"]]["args"]          args=mastertrees[kinfo["mastertree"]]["args"]
Line 266 
Line 302 
         log("fetching source archive %s" % args)          log("fetching source archive %s" % args)
         mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])          mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])
   
         if os.system("wget -c --output-document=%s %s" % (mysourcefile,args)):          if os.spawnlp(os.P_WAIT, "wget -c --output-document=%s %s" % (mysourcefile,args)):
                 deathbyerror("%s : unable to download source file %s, aborting." % (kinfo["name"], args))                  raise BuildError("unable to download source file %s, aborting." %  args)
   
         log("decompressing source file %s" % mysourcefile,kinfo)          log("decompressing source file %s" % mysourcefile,kinfo)
   
Line 282 
Line 318 
                 trash=pfd.read(4096)                  trash=pfd.read(4096)
   
         if pfd.close():          if pfd.close():
                 deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"]))                  raise BuildError("unable to decompress source file, aborting.")
   
         fnames=split(data)          fnames=split(data)
         name=fnames[0]          name=fnames[0]
   
         os.system("rm -rf %s" % kinfo["workdir"])          os.spawnlp(os.P_WAIT, "rm -rf %s" % kinfo["workdir"])
         move(work + "/" + dirname, kinfo["workdir"])          move(work + "/" + dirname, kinfo["workdir"])
   
         #restore the previous working ectory          #restore the previous working ectory
Line 305 
Line 341 
                         mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)                          mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)
   
                         # download pathfile or bail                          # download pathfile or bail
                         if os.system("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)):                          if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)):
                                 deathbyerror("%s : unable to download patchfile %s, aborting." % (kinfo["name"],patch))                                  raise BuildError("unable to download patchfile %s, aborting." % patch)
   
                         # we have a patch file, apply it or bail                          # we have a patch file, apply it or bail
                         log("perfoming patch with %s" % patch,kinfo)                          log("perfoming patch with %s" % patch,kinfo)
                         patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch)                          patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch)
                         log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)                          log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)
                         chdir(kinfo["workdir"],kinfo)                          chdir(kinfo["workdir"],kinfo)
                         if os.system(patchcommand):                          if os.spawnlp(os.P_WAIT, patchcommand):
                                 deathbyerror("%s : patchfile %s failed, aborting.  See patch log for details." % (kinfo["name"],patch))                                  raise BuildError("patchfile %s failed, aborting.  See patch log for details." % patch)
         else:          else:
                 log("no patchfiles, continuing",kinfo)                  log("no patchfiles, continuing",kinfo)
   
Line 325 
Line 361 
                 log("fetching config file",kinfo)                  log("fetching config file",kinfo)
                 myconfigfile="%s/%s.config" % (configdir,kinfo["name"])                  myconfigfile="%s/%s.config" % (configdir,kinfo["name"])
   
                 if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):                  if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
                         deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"]))                          raise BuildError("unable to download configfile, aborting.")
   
                 log("copying config file to %s/.config" % kinfo["workdir"],kinfo)                  log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
                 copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])                  copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])
Line 335 
Line 371 
 def gkb_runmake(command, kinfo, premake, makeopts):  def gkb_runmake(command, kinfo, premake, makeopts):
         """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""          """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""
         log("running make %s" % command,kinfo)          log("running make %s" % command,kinfo)
         if os.system("%s make %s %s > %s/%s/make-%s.log 2>&1" % (premake, makeopts, command, logdir, kinfo["name"], command)):          if os.spawnlp(os.P_WAIT, "%s make %s %s > %s/%s/make-%s.log 2>&1" % (premake, makeopts, command, logdir, kinfo["name"], command)):
                 deathbyerror("%s : unable to run make %s, aborting." % (kinfo["name"],command))                  raise BuildError("unable to run make %s, aborting." % command)
         # Gerk comment:  
         # see other comment above about shunting 2>log  
   
 def gkb_parsexml(name):  def gkb_parsexml(name):
         """parse the xml build file into mastertrees and builds"""          """parse the xml build file into mastertrees and builds"""
Line 353 
Line 387 
         p.StartElementHandler = start_element          p.StartElementHandler = start_element
         p.ParseFile(open(name))          p.ParseFile(open(name))
   
   def krn_upload(file, type, version, kinfo):
           """upload the indicated file to the distribution site (kernel archives)"""
   
           if type=="kernel":
                   log("uploading kernel version "+version+" to "+msite,kinfo)
           elif type=="source":
                   log("uploading source "+kinfo["mastertree"]+" to "+msite,kinfo)
           else:
                   raise BuildException, "invalid file upload type: " + type
   
           forms = ParseResponse(urlopen(msite+"/upload.html"))
           form = forms[0]
   
           form["host"] = host
           form["pass"] = passwd
           form["type"] = type
           form["tree"] = kinfo["mastertree"]
           form["build"] = kinfo["name"]
           form["version"] = version
   
           form.add_file(open(file), "application/x-bzip2", os.path.basename(file))
   
           # form.click() returns a urllib2.Request object
           # (see HTMLForm.click.__doc__ if you don't have urllib2)
           response = urlopen(form.click("cmd"))
   
           if debug:
                   print response.geturl()
                   print response.info()  # headers
                   print response.read()  # body
   
           log("response: " + response.read(),kinfo)
   
           response.close()
   
 def main():  def main():
         """ main program gets executed here """          """ main program gets executed here """
   
Line 372 
Line 441 
         verifydir("%s/work" % buildroot) # build dir, make sure it exists          verifydir("%s/work" % buildroot) # build dir, make sure it exists
   
         # download the build jobs from the master site          # download the build jobs from the master site
         if os.system("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):          if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):
                 deathbyerror("Unable to download build config from master site %s, aborting." % msite)                  deathbyerror("Unable to download build config from master site %s, aborting." % msite)
   
         # sets up 'mastertrees' and 'builds' dicts          # sets up 'mastertrees' and 'builds' dicts
Line 382 
Line 451 
                 myworkdir="%s/%s" % (workdir,bdict["mastertree"])                  myworkdir="%s/%s" % (workdir,bdict["mastertree"])
                 bdict["workdir"]=myworkdir                  bdict["workdir"]=myworkdir
   
                   try:
                 # for now just call the build                  # for now just call the build
                 gkb_build(root, bdict)                  gkb_build(root, bdict)
                   except BuildError, e:
                           log(e.message, bdict)
   
         endtime=getoutput("date +'%R:%S %Z'")          endtime=getoutput("date +'%R:%S %Z'")
   
Line 391 
Line 463 
   
 # and finally, call the mainloop to execute  # and finally, call the mainloop to execute
 main()  main()
   

Generate output suitable for use with a patch program
Legend:
Removed from v.1.1.1.1  
changed lines
  Added in v.1.11
Tobias McNulty

Powered by ViewCVS 1.0-dev 		(Powered by ViewCVS)

ViewCVS and CVS Help