Powered by ViewCVS 1.0-dev
changed to use spawnlp instead of system (avoid strange KeyboardInterrupt behavior)
#!/usr/bin/python
#
# GKB - GNU Kernel Builder
# Copyright (C) 2003-2004 Tobias McNulty, Mark Guertin
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Ported from PHP version Copyright (c) Tobias McNulty 2000-2003
import sys, os, ConfigParser
import xml.parsers.expat
import md5
from ftplib import FTP
from commands import getoutput
from shutil import copy, move
from string import split
from urllib2 import urlopen
from ClientForm import ParseResponse
debug=0
clean=0
dont_build=1
verbose=1
# parse host-specific configuration information from gkb.cfg
config = ConfigParser.ConfigParser()
config.readfp(open('gkb-host.cfg'))
buildroot=config.get("options","buildroot")
host=config.get("options","host")
passwd=config.get("options","passwd")
# this one is passed verbatim before the make command. might be useful for alternate toolchains, etc
# example: premake="HOST=powerpc-unknown-linux-gnu"
premake=config.get("options","premake")
# make options, passed verbatim after make command
makeopts=config.get("options","makeopts")
# parse manager-related configuration from gkb-manager.cfg
config.readfp(open('gkb-manager.cfg'))
msite=config.get('options','msite')
# these values will likely stay fairly static across different builds
patchdir="%s/patches" % buildroot # full path to patch file dir
configdir="%s/configs" % buildroot # path to where configs are to be stored
logdir="%s/logs" % buildroot # the logs will go here
bindir="%s/bin" % buildroot # the directory that binaries built will be sent for pkging
workdir="%s/work" % buildroot # work directory, where builds take place
# setup the global dicts to hold build data, imported from xml
mastertrees={}
builds={}
class Error(Exception):
"""Base class for exceptions in this module."""
pass
class BuildError(Error):
"""Exception raised for build-level errors.
Attributes:
message -- explanation of the error
"""
def __init__(self, message):
self.message = message
class FatalError(Error):
"""Exception for fatal errors that prevent program continuation.
Attributes:
message -- explanation of why the specific transition is not allowed
"""
def __init__(self, previous, next, message):
self.message = message
def printverbose(myoutput):
"""Supporting method to output info to stdout if verbosity level is set"""
if verbose==1:
print myoutput
def log(text,kinfo=0):
if kinfo:
printverbose(kinfo["name"] + " : " + text)
else:
printverbose("gkb : " + text)
def verifydir(mydir,kinfo=0):
"""Supporting method to verify a given exists, if not it will create it"""
log("verifying directory " + mydir,kinfo)
if not os.path.isdir(mydir):
log("%s doesn't exist, creating" % mydir,kinfo)
os.mkdir(mydir)
return mydir
def chdir(mydir,kinfo):
verifydir(mydir,kinfo)
log("entering "+mydir,kinfo)
os.chdir(mydir)
def verifyfile(myfile,kinfo):
"""Supporting method to verify file exists, if not it will exit with error"""
if not os.path.isfile(myfile):
raise BuildError("cannot find file expected at %s, exiting" % myfile)
return myfile
def md5sum(fileobj):
"""calculates the md5sum for fileobj and returns it in hexadecimal"""
md = md5.new()
data = fileobj.read(8192)
while (data):
md.update(data)
data = fileobj.read(8192)
digest = md.hexdigest()
return digest
def krn_querymgr(command,kinfo):
"""queries the build host manager with a variety of commands, such as checkout, checkin, etc."""
version=krn_localversion(kinfo)
log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo)
result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s\"" % (msite,command,host,passwd,kinfo["name"],version))
log("result: '"+result+"'",kinfo)
return result=="1"
def krn_localversion(kinfo):
"""Checks the version of the local source tree specified in kinfo"""
version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """)
return version
def gkb_build(root,kinfo):
"""performs actual kernel builds given root, kinfo (tuple containing relevant data) , premake and makeopts"""
# make sure needed directoriess (for this build) exist, if not create them
verifydir("%s/%s" % (logdir, kinfo["name"]),kinfo)
verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo)
# sync the source to make sure we are up to date ...
log("calling gkb_getsource",kinfo)
gkb_getsource(kinfo)
# go into the work directory
chdir(workdir)
# archive the clean source for later uploading
os.spawnlp(os.P_WAIT, "tar xjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"])
# now we'll go into the tree's work dir
chdir(kinfo["workdir"],kinfo)
# check for patches and apply if necessary
gkb_patch(kinfo)
if kinfo["type"] == "kernel24":
krn_build24(kinfo)
elif kinfo["type"] == "kernel26":
krn_build24(kinfo) #run kernel24 for now, change later
# Gerk comment:
# Do we need to change this with 24/26? we probably don't ... I'd rather see us define the list of make targets
# i.e. "dep","clean","vmlinux","modules","modules_install" or "clean","all","modules_install" for 2.6
# also of note, make all is not what we will always want with 2.6 kernels ... i.e. zImage.prep, zImage.chrp, etc
# also for backward compat they will continue to support calling things individual .. the makefile just does
# this for us with 'all'
def krn_build24(kinfo):
"""build a kernel, version 2.4.x"""
if krn_querymgr("checkout",kinfo):
try:
myversion=krn_localversion(kinfo)
# fetch and cp the config file to work/.config
krn_config(kinfo)
# **Note** : we set the preprocessing command to premake inline instead
# of globally as it is only needed in this target
if dont_build==0:
gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts)
# give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature
if (clean==1 and dont_build==0):
gkb_runmake("clean", kinfo, premake, makeopts)
if dont_build==0:
gkb_runmake("dep", kinfo, premake, makeopts)
gkb_runmake(kinfo["binname"], kinfo, premake, makeopts)
# We should check to see if binary built ok, if not bail out
mybindir=bindir+"/linux-"+kinfo["name"]+"-"+myversion
verifydir(mybindir,kinfo)
verifydir(mybindir+"/boot",kinfo)
kbinloc=kinfo["workdir"]+"/"+kinfo["binpath"]+"/"+kinfo["binname"]
if verifyfile(kbinloc,kinfo):
# the binary exists, so let's cp it to bin...
copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)
else:
# the binary is not there, inform user and bail out with error
raise BuildError("%s is not present, assuming build failure and exiting. See log for details." % kbinloc)
# now that we know he binary built, let's continue
if dont_build==0:
gkb_runmake("modules",kinfo, premake, makeopts)
# **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead
# of globally as it is only needed in this target
gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))
# compress and upload source archive
chdir(kinfo["workdir"]+"/..",kinfo)
archive_name = "src-%s.tar.bz2" % kinfo["mastertree"],
# compress and upload kernel binary
chdir(bindir,kinfo)
archive_name = "linux-%s-%s.tar.bz2" % (kinfo["name"], myversion)
log("compressing binary archive "+archive_name,kinfo)
if os.spawnlp(os.P_WAIT, "tar cjf "+archive_name+" "+os.path.basename(mybindir)):
raise BuildError("failed to `tar cjf %s`" % archive_name)
os.spawnlp(os.P_WAIT, "rm -rf %s" % mybindir)
krn_upload(archive_name,"kernel",myversion,kinfo)
krn_querymgr("checkin",kinfo)
if (os.fork() == 0):
#in child
krn_upload(workdir + "/" + kinfo["mastertree"] + ".tar.bz2", "source", myversion, kinfo)
except BuildError, e:
log(e.message, kinfo)
krn_querymgr("checkin",kinfo)
except:
krn_querymgr("checkin",kinfo)
raise
def krn_build26(kinfo):
"""build a kernel, version 2.6"""
# source get routines
def gkb_getsource(kinfo):
"""method to perform source sync on demand, currenty only supports rsync, but others can be added"""
method=mastertrees[kinfo["mastertree"]]["method"]
if method == "rsync":
get_rsync(kinfo)
elif method == "wget":
get_wget(kinfo)
elif method == "vanilla":
get_vanilla(kinfo)
def get_rsync(kinfo):
"""sync the source using rsync"""
args=mastertrees[kinfo["mastertree"]]["args"]
if clean==1:
syncoptions="rsync -azv --delete"
else:
syncoptions="rsync -azv"
syncline=args+" "+kinfo["workdir"]
log("running rsync : %s %s" % (syncoptions, syncline),kinfo)
if os.spawnlp(os.P_WAIT, "%s %s > %s/%s/rsync.log 2>&1" % (syncoptions, syncline, logdir, kinfo["name"])):
raise BuildError("sync failed, tried %s. See log for details." % syncline)
def get_wget(kinfo):
#needs work
"""sync the source using wget and a tar.bz2"""
args=mastertrees[kinfo["mastertree"]]["args"]
log("fetching source archive " % args,kinfo)
mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])
if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
raise BuildError("unable to download configfile, aborting.")
log("decompressing source file",kinfo)
if os.spawnlp(os.P_WAIT, "tar xjf " % (myconfigfile,msite,kinfo["name"])):
raise BuildError("unable to decompress source file, aborting.")
def get_vanilla(kinfo):
args=mastertrees[kinfo["mastertree"]]["args"]
log("fetching source archive %s" % args)
mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])
if os.spawnlp(os.P_WAIT, "wget -c --output-document=%s %s" % (mysourcefile,args)):
raise BuildError("unable to download source file %s, aborting." % args)
log("decompressing source file %s" % mysourcefile,kinfo)
#save the current working ectory
cwd=getoutput("pwd")
chdir(workdir,kinfo)
pfd=os.popen("tar vxjf %s" % mysourcefile,"r")
trash=data=pfd.read(255)
while trash:
trash=pfd.read(4096)
if pfd.close():
raise BuildError("unable to decompress source file, aborting.")
fnames=split(data)
name=fnames[0]
os.spawnlp(os.P_WAIT, "rm -rf %s" % kinfo["workdir"])
move(work + "/" + dirname, kinfo["workdir"])
#restore the previous working ectory
chdir(cwd,kinfo)
def gkb_patch(kinfo):
"""method that applies patch found in kinfo config files for running kernel build"""
if kinfo["patches"]:
patches=split(kinfo["patches"],";")
for patch in patches:
log("downloading patch %s" % patch,kinfo)
#set a name for this patch file
mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)
# download pathfile or bail
if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)):
raise BuildError("unable to download patchfile %s, aborting." % patch)
# we have a patch file, apply it or bail
log("perfoming patch with %s" % patch,kinfo)
patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch)
log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)
chdir(kinfo["workdir"],kinfo)
if os.spawnlp(os.P_WAIT, patchcommand):
raise BuildError("patchfile %s failed, aborting. See patch log for details." % patch)
else:
log("no patchfiles, continuing",kinfo)
def krn_config(kinfo):
"""method to fetch and place config file for running kernel build"""
if kinfo["config"]==1:
log("fetching config file",kinfo)
myconfigfile="%s/%s.config" % (configdir,kinfo["name"])
if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
raise BuildError("unable to download configfile, aborting.")
log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])
# Methods to support build()
def gkb_runmake(command, kinfo, premake, makeopts):
"""Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""
log("running make %s" % command,kinfo)
if os.spawnlp(os.P_WAIT, "%s make %s %s > %s/%s/make-%s.log 2>&1" % (premake, makeopts, command, logdir, kinfo["name"], command)):
raise BuildError("unable to run make %s, aborting." % command)
def gkb_parsexml(name):
"""parse the xml build file into mastertrees and builds"""
def start_element(name, attrs):
#print "In start_element: %s %s" % (name, attrs)
if name=="mastertree":
mastertrees[attrs["name"]]=attrs
elif name=="build":
builds[attrs["name"]]=attrs
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = start_element
p.ParseFile(open(name))
def krn_upload(file, type, version, kinfo):
"""upload the indicated file to the distribution site (kernel archives)"""
if type=="kernel":
log("uploading kernel version "+version+" to "+msite,kinfo)
elif type=="source":
log("uploading source "+kinfo["mastertree"]+" to "+msite,kinfo)
else:
raise BuildException, "invalid file upload type: " + type
forms = ParseResponse(urlopen(msite+"/upload.html"))
form = forms[0]
form["host"] = host
form["pass"] = passwd
form["type"] = type
form["tree"] = kinfo["mastertree"]
form["build"] = kinfo["name"]
form["version"] = version
form.add_file(open(file), "application/x-bzip2", os.path.basename(file))
# form.click() returns a urllib2.Request object
# (see HTMLForm.click.__doc__ if you don't have urllib2)
response = urlopen(form.click("cmd"))
if debug:
print response.geturl()
print response.info() # headers
print response.read() # body
log("response: " + response.read(),kinfo)
response.close()
def main():
""" main program gets executed here """
# get our working ectory
root=getoutput("pwd")
today=getoutput("date +%D")
buildtime=getoutput("date +'%R:%S %Z'")
print "GKB started %s %s" % (today,buildtime)
# verify the existence important directories, and create if necessary
verifydir(logdir)
verifydir(configdir)
verifydir(patchdir)
verifydir(bindir)
verifydir("%s/work" % buildroot) # build dir, make sure it exists
# download the build jobs from the master site
if os.spawnlp(os.P_WAIT, "wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):
deathbyerror("Unable to download build config from master site %s, aborting." % msite)
# sets up 'mastertrees' and 'builds' dicts
gkb_parsexml('gkb.xml')
for bdict in builds.values():
myworkdir="%s/%s" % (workdir,bdict["mastertree"])
bdict["workdir"]=myworkdir
try:
# for now just call the build
gkb_build(root, bdict)
except BuildError, e:
log(e.message, bdict)
endtime=getoutput("date +'%R:%S %Z'")
print "GKB finished %s %s" % (today,endtime)
# and finally, call the mainloop to execute
main()
|
Tobias McNulty Powered by ViewCVS 1.0-dev |
ViewCVS and CVS Help |