[cvs] / gkb / gkb.py  

cvs: gkb/gkb.py

Diff for /gkb/gkb.py between version 1.18 and 1.36

version 1.18, Sun Aug 15 03:44:06 2004 UTC version 1.36, Sun Dec 19 03:24:30 2004 UTC
Line 19 
Line 19 
 #  #
 # Ported from PHP version Copyright (c) Tobias McNulty 2000-2003  # Ported from PHP version Copyright (c) Tobias McNulty 2000-2003
   
 import sys, os, ConfigParser  import sys, os, ConfigParser, popen2
 import xml.parsers.expat  import xml.parsers.expat
 import md5  import md5, string
 from ftplib import FTP  from ftplib import FTP
 from commands import getoutput  from commands import getoutput
 from shutil import copy, move  from shutil import copy, move
Line 29 
Line 29 
 from urllib2 import urlopen  from urllib2 import urlopen
 from ClientForm import ParseResponse  from ClientForm import ParseResponse
   
   
 debug=0  debug=0
 clean=0  clean=1
 dont_build=1  dont_build=0
 verbose=1  verbose=1
   
 # parse host-specific configuration information from gkb.cfg  # parse host-specific configuration information from gkb.cfg
Line 91 
Line 92 
         if verbose==1:          if verbose==1:
                 print myoutput                  print myoutput
   
   def cleantext(text):
           """clean a string of passwords, etc. to make it suitable for logging"""
           result = string.replace(text,passwd,"XXXXXXXX")
           return result
   
 def linkpipes(input, output):  def linkpipes(input, output):
           """link two pipes together, writing all the data from input to output"""
         data = input.read(2048)          data = input.read(2048)
         while data:          while data:
                 output.write(data)                  output.write(data)
                 data = input.read(2048)                  data = input.read(2048)
   
 def runcmd(cmd, outfile=None, append=False, infile=None):  def runcmd(cmd, kinfo=None, outfile=None, append=False, infile=None):
         log("running " + cmd)          """run the specified command, with the optional input and output documents"""
           log("  \_ running " + cleantext(cmd), kinfo)
   
           p4obj = popen2.Popen4(cmd)
   
         (pin, pouterr) = os.popen4(cmd)          pin = p4obj.tochild
           pouterr = p4obj.fromchild
   
         if infile != None:          if infile != None:
                 inobj = open(infile, "r")                  inobj = open(infile, "r")
Line 109 
Line 120 
   
         if outfile != None:          if outfile != None:
                 if append:                  if append:
                         print "Alert: appending"  
                         mode = "a"                          mode = "a"
                 else:                  else:
                         mode = "w"                          mode = "w"
Line 118 
Line 128 
                 linkpipes(pouterr,outobj)                  linkpipes(pouterr,outobj)
                 outobj.close()                  outobj.close()
   
         err = pouterr.close()          err = p4obj.wait()
   
         if err == None:          if err == None:
                 err = 0                  err = 0
   
         log("result code: %i" %  err)          log("    \_ exit code: %i" %  err, kinfo)
   
         return err          return err
   
 def log(text,kinfo=0):  def log(text,kinfo=0):
Line 136 
Line 147 
         """Supporting method to verify a given  exists, if not it will create it"""          """Supporting method to verify a given  exists, if not it will create it"""
         log("verifying directory " + mydir,kinfo)          log("verifying directory " + mydir,kinfo)
         if not os.path.isdir(mydir):          if not os.path.isdir(mydir):
                 log("%s doesn't exist, creating" % mydir,kinfo)                  log("  \_ %s doesn't exist, creating" % mydir,kinfo)
                 os.mkdir(mydir)                  os.mkdir(mydir)
         return mydir          return mydir
   
Line 162 
Line 173 
         digest = md.hexdigest()          digest = md.hexdigest()
         return digest          return digest
   
 def krn_querymgr(command,kinfo):  def krn_querymgr(command,kinfo,extra=""):
         """queries the build host manager with a variety of commands, such as checkout, checkin, etc."""          """queries the build host manager with a variety of commands, such as checkout, checkin, etc."""
         version=krn_localversion(kinfo)          version=krn_localversion(kinfo)
         log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo)          log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo)
         result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s\"" % (msite,command,host,passwd,kinfo["name"],version))          result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s%s\"" % (msite,command,host,passwd,kinfo["name"],version,extra))
         log("result: '"+result+"'",kinfo)          log("  \_ result: '"+result+"'",kinfo)
         return result=="1"          return result=="1"
   
   def krn_modulesenabled(kinfo):
           """Checks the source to see if modules are enabled"""
           output=getoutput("""awk -F '=' '/^CONFIG_MODULES/{v=$2} END { printf("%s\\n", v) }' """ + kinfo["workdir"] + """/.config | sed "s/ //g" """)
           if (output=='y'):
                   result=1
           else:
                   result=0
   
           return result
   
 def krn_localversion(kinfo):  def krn_localversion(kinfo):
         """Checks the version of the local source tree specified in kinfo"""          """Checks the version of the local source tree specified in kinfo"""
         version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """)          version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """)
Line 184 
Line 205 
         verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo)          verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo)
   
         # sync the source to make sure we are up to date ...          # sync the source to make sure we are up to date ...
         log("calling gkb_getsource",kinfo)          log("fetching latest source",kinfo)
         gkb_getsource(kinfo)          gkb_getsource(kinfo)
   
         # go into the work directory          # go into the work directory
         chdir(workdir, kinfo)          #chdir(workdir, kinfo)
   
         # archive the clean source for later uploading          # archive the clean source for later uploading
         log("archiving source to " + kinfo["mastertree"] + ".tar.bz2", kinfo)          #log("archiving source to " + kinfo["mastertree"] + ".tar.bz2", kinfo)
         runcmd("tar cjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"])          #runcmd("tar cjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"], kinfo)
   
         # now we'll go into the tree's work dir          # now we'll go into the tree's work dir
         chdir(kinfo["workdir"],kinfo)          chdir(kinfo["workdir"],kinfo)
Line 245 
Line 266 
                         if verifyfile(kbinloc,kinfo):                          if verifyfile(kbinloc,kinfo):
                                 # the binary exists, so let's cp it to bin...                                  # the binary exists, so let's cp it to bin...
                                 copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)                                  copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)
                                   copy("%s/System.map" % kinfo["workdir"],mybindir+"/boot/System.map-"+myversion)
                         else:                          else:
                                 # the binary is not there, inform user and bail out with error                                  # the binary is not there, inform user and bail out with error
                                 raise BuildError("%s is not present, assuming build failure and exiting.  See log for details." % kbinloc)                                  raise BuildError("%s is not present, assuming build failure and exiting.  See log for details." % kbinloc)
   
                           if krn_modulesenabled(kinfo):
                         # now that we know he binary built, let's continue                          # now that we know he binary built, let's continue
                         if dont_build==0:                          if dont_build==0:
                                 gkb_runmake("modules",kinfo, premake, makeopts)                                  gkb_runmake("modules",kinfo, premake, makeopts)
Line 256 
Line 279 
                         # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead                          # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead
                         # of globally as it is only needed in this target                          # of globally as it is only needed in this target
                         gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))                          gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))
                           else:
                                   log("skipping make modules (disabled in .config)",kinfo)
   
                         # compress and upload source archive                          # compress and upload source archive
                         chdir(kinfo["workdir"]+"/..",kinfo)                          chdir(kinfo["workdir"]+"/..",kinfo)
Line 267 
Line 292 
   
                         log("compressing binary archive "+archive_name,kinfo)                          log("compressing binary archive "+archive_name,kinfo)
   
                         if runcmd("tar cjf "+archive_name+" "+os.path.basename(mybindir)):                          if runcmd("tar cjf "+archive_name+" "+os.path.basename(mybindir), kinfo):
                                 raise BuildError("failed to `tar cjf %s`" %  archive_name)                                  raise BuildError("failed to `tar cjf %s`" %  archive_name)
   
                         runcmd("rm -rf %s" % mybindir)                          runcmd("rm -rf %s" % mybindir, kinfo)
   
                         krn_upload(archive_name,"kernel",myversion,kinfo)                          krn_upload(archive_name,"kernel",myversion,kinfo)
                         krn_querymgr("checkin",kinfo)                          krn_querymgr("checkin",kinfo)
   
                         if (os.fork() == 0):  #                       if os.fork() == 0:
                                 #in child  #                               #in child
                                 krn_upload(workdir + "/" + kinfo["mastertree"] + ".tar.bz2", "source", myversion, kinfo)  #                               try:
   #                                       # sync the source to make sure we are up to date ...
   #                                       log("re-fetching latest source",kinfo)
   #                                       gkb_getsource(kinfo)
   #
   #                                       # go into the work directory
   #                                       chdir(workdir, kinfo)
   #
   #                                       # archive the clean source for later uploading
   #                                       log("archiving source to " + kinfo["mastertree"] + ".tar.bz2", kinfo)
   #                                       runcmd("tar cjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"], kinfo)
   #
   #                                       krn_upload(workdir + "/" + kinfo["mastertree"] + ".tar.bz2", "source", myversion, kinfo)
   #                               finally:
   #                                       sys.exit(0)
   
                           log("returning source directory to clean state",kinfo)
                           gkb_getsource(kinfo)
   
                 except BuildError, e:                  except BuildError, e:
                         log(e.message, kinfo)                          log(e.message, kinfo)
                         krn_querymgr("checkin",kinfo)                          krn_querymgr("checkin",kinfo,"&failed=%s" % krn_localversion(kinfo))
                 except:                  except:
                         krn_querymgr("checkin",kinfo)                          krn_querymgr("checkin",kinfo,"&failed=%s" % krn_localversion(kinfo))
                         raise                          raise
   
 def krn_build26(kinfo):  def krn_build26(kinfo):
Line 313 
Line 355 
   
         syncline=args+" "+kinfo["workdir"]          syncline=args+" "+kinfo["workdir"]
   
         log("running rsync : %s %s" % (syncoptions, syncline),kinfo)          #logging now in runcmd
         if runcmd("%s %s" % (syncoptions, syncline), "%s/%s/rsync.log" % (logdir, kinfo["name"])):          #log("running rsync : %s %s" % (syncoptions, syncline),kinfo)
           if runcmd("%s %s" % (syncoptions, syncline), kinfo, "%s/%s/rsync.log" % (logdir, kinfo["name"])):
                 raise BuildError("sync failed, tried %s.  See log for details." %  syncline)                  raise BuildError("sync failed, tried %s.  See log for details." %  syncline)
   
 def get_wget(kinfo):  def get_wget(kinfo):
Line 325 
Line 368 
         log("fetching source archive " % args,kinfo)          log("fetching source archive " % args,kinfo)
         mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])          mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])
   
         if runcmd("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):          if runcmd("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"]), kinfo):
                 raise BuildError("unable to download configfile, aborting.")                  raise BuildError("unable to download configfile, aborting.")
   
         log("decompressing source file",kinfo)          log("decompressing source file",kinfo)
         if runcmd("tar xjf " % (myconfigfile,msite,kinfo["name"])):          if runcmd("tar xjf " % (myconfigfile,msite,kinfo["name"]), kinfo):
                 raise BuildError("unable to decompress source file, aborting.")                  raise BuildError("unable to decompress source file, aborting.")
   
 def get_vanilla(kinfo):  def get_vanilla(kinfo):
Line 338 
Line 381 
         log("fetching source archive %s" % args)          log("fetching source archive %s" % args)
         mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])          mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])
   
         if runcmd("wget -c --output-document=%s %s" % (mysourcefile,args)):          if runcmd("wget -c --output-document=%s %s" % (mysourcefile,args), kinfo):
                 raise BuildError("unable to download source file %s, aborting." %  args)                  raise BuildError("unable to download source file %s, aborting." %  args)
   
         log("decompressing source file %s" % mysourcefile,kinfo)          log("decompressing source file %s" % mysourcefile,kinfo)
Line 359 
Line 402 
         fnames=split(data)          fnames=split(data)
         name=fnames[0]          name=fnames[0]
   
         runcmd("rm -rf %s" % kinfo["workdir"])          runcmd("rm -rf %s" % kinfo["workdir"], kinfo)
         move(work + "/" + dirname, kinfo["workdir"])          move(work + "/" + dirname, kinfo["workdir"])
   
         #restore the previous working ectory          #restore the previous working ectory
Line 377 
Line 420 
                         mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)                          mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)
   
                         # download pathfile or bail                          # download pathfile or bail
                         if runcmd("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)):                          if runcmd("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch), kinfo):
                                 raise BuildError("unable to download patchfile %s, aborting." % patch)                                  raise BuildError("unable to download patchfile %s, aborting." % patch)
   
                         # we have a patch file, apply it or bail                          # we have a patch file, apply it or bail
Line 388 
Line 431 
                         #log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)                          #log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)
   
                         chdir(kinfo["workdir"],kinfo)                          chdir(kinfo["workdir"],kinfo)
                         if runcmd("patch -p1", "%s/%s/patch-%s.log" % (logdir,kinfo["name"],patch), False, mypatchfile):                          if runcmd("patch -p1", kinfo, "%s/%s/patch-%s.log" % (logdir,kinfo["name"],patch), False, mypatchfile):
                                 raise BuildError("patchfile %s failed, aborting.  See patch log for details." % patch)                                  raise BuildError("patchfile %s failed, aborting.  See patch log for details." % patch)
         else:          else:
                 log("no patchfiles, continuing",kinfo)                  log("no patchfiles, continuing",kinfo)
Line 396 
Line 439 
 def krn_config(kinfo):  def krn_config(kinfo):
         """method to fetch and place config file for running kernel build"""          """method to fetch and place config file for running kernel build"""
   
         if kinfo["config"]==1:          if kinfo["configpath"]:
                   log("using config file %s" % kinfo["configpath"],kinfo)
                   myconfigfile="%s/%s" % (kinfo["workdir"],kinfo["configpath"])
   
                   log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
                   copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])
   
           elif kinfo["config"]:
                 log("fetching config file",kinfo)                  log("fetching config file",kinfo)
                 myconfigfile="%s/%s.config" % (configdir,kinfo["name"])                  myconfigfile="%s/%s.config" % (configdir,kinfo["name"])
   
                 if runcmd("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):                  if runcmd("wget --quiet --output-document=%s %s/configs/%s/%s" % (myconfigfile,msite,kinfo["name"],kinfo["config"]), kinfo):
                         raise BuildError("unable to download configfile, aborting.")                          raise BuildError("unable to download configfile, aborting.")
   
                 log("copying config file to %s/.config" % kinfo["workdir"],kinfo)                  log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
Line 410 
Line 460 
 def gkb_runmake(command, kinfo, premake, makeopts):  def gkb_runmake(command, kinfo, premake, makeopts):
         """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""          """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""
         log("running make %s" % command,kinfo)          log("running make %s" % command,kinfo)
         if runcmd("%s make %s %s" % (premake, makeopts, command), "%s/%s/make-%s.log" % (logdir, kinfo["name"], command)):  
           # check to see if we need to run a simple make
           # for some reason 2.2 kernels can't handle premake, makeopts, etc.
           localversion = krn_localversion(kinfo)
   
           if (string.find(localversion,"2.2") == 0):
                   if runcmd("%s make %s" % (premake,command), kinfo, "%s/%s/make-%s.log" % (logdir, kinfo["name"], command)):
                           raise BuildError("unable to run make %s, aborting." % command)
           else:
                   if runcmd("%s make %s %s" % (premake, makeopts, command), kinfo, "%s/%s/make-%s.log" % (logdir, kinfo["name"], command)):
                 raise BuildError("unable to run make %s, aborting." % command)                  raise BuildError("unable to run make %s, aborting." % command)
   
 def gkb_parsexml(name):  def gkb_parsexml(name):
Line 457 
Line 516 
                 print response.info()  # headers                  print response.info()  # headers
                 print response.read()  # body                  print response.read()  # body
   
         log("response: " + response.read(),kinfo)          log("  \_ response: " + response.read(),kinfo)
   
         response.close()          response.close()
   
Line 470 
Line 529 
         today=getoutput("date +%D")          today=getoutput("date +%D")
         buildtime=getoutput("date +'%R:%S %Z'")          buildtime=getoutput("date +'%R:%S %Z'")
   
         print "GKB started %s %s" % (today,buildtime)          print "GNU Kernel Builder started %s %s" % (today,buildtime)
   
         # verify the existence important directories, and create if necessary          # verify the existence important directories, and create if necessary
         verifydir(logdir)          verifydir(logdir)
Line 480 
Line 539 
         verifydir("%s/work" % buildroot) # build dir, make sure it exists          verifydir("%s/work" % buildroot) # build dir, make sure it exists
   
         # download the build jobs from the master site          # download the build jobs from the master site
           log("fetching build jobs from master site")
         if runcmd("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):          if runcmd("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):
                 raise FatalError, "Unable to download build config from master site %s, aborting." % msite                  raise FatalError, "Unable to download build jobs from master site %s, aborting." % msite
   
         # sets up 'mastertrees' and 'builds' dicts          # sets up 'mastertrees' and 'builds' dicts
         gkb_parsexml('gkb.xml')          gkb_parsexml('gkb.xml')

Generate output suitable for use with a patch program
Legend:
Removed from v.1.18  
changed lines
  Added in v.1.36
Tobias McNulty

Powered by ViewCVS 1.0-dev 		(Powered by ViewCVS)

ViewCVS and CVS Help