Powered by ViewCVS 1.0-dev
| 1 : | tmcnulty | 1.1 | #!/usr/bin/python |
| 2 : | # | ||
| 3 : | # GKB - GNU Kernel Builder | ||
| 4 : | tmcnulty | 1.2 | # Copyright (C) 2003-2004 Tobias McNulty, Mark Guertin |
| 5 : | tmcnulty | 1.1 | # |
| 6 : | # This program is free software; you can redistribute it and/or modify | ||
| 7 : | # it under the terms of the GNU General Public License as published by | ||
| 8 : | # the Free Software Foundation; either version 2 of the License, or | ||
| 9 : | # (at your option) any later version. | ||
| 10 : | # | ||
| 11 : | # This program is distributed in the hope that it will be useful, | ||
| 12 : | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 13 : | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 14 : | # GNU General Public License for more details. | ||
| 15 : | # | ||
| 16 : | # You should have received a copy of the GNU General Public License | ||
| 17 : | # along with this program; if not, write to the Free Software | ||
| 18 : | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
| 19 : | # | ||
| 20 : | # Ported from PHP version Copyright (c) Tobias McNulty 2000-2003 | ||
| 21 : | |||
| 22 : | import sys, os, ConfigParser | ||
| 23 : | import xml.parsers.expat | ||
| 24 : | import md5 | ||
| 25 : | from ftplib import FTP | ||
| 26 : | from commands import getoutput | ||
| 27 : | from shutil import copy, move | ||
| 28 : | from string import split | ||
| 29 : | from urllib2 import urlopen | ||
| 30 : | from ClientForm import ParseResponse | ||
| 31 : | |||
| 32 : | debug=0 | ||
| 33 : | clean=1 | ||
| 34 : | verbose=1 | ||
| 35 : | |||
| 36 : | # parse host-specific configuration information from gkb.cfg | ||
| 37 : | config = ConfigParser.ConfigParser() | ||
| 38 : | config.readfp(open('gkb-host.cfg')) | ||
| 39 : | buildroot=config.get("options","buildroot") | ||
| 40 : | host=config.get("options","host") | ||
| 41 : | passwd=config.get("options","passwd") | ||
| 42 : | |||
| 43 : | # this one is passed verbatim before the make command. might be useful for alternate toolchains, etc | ||
| 44 : | # example: premake="HOST=powerpc-unknown-linux-gnu" | ||
| 45 : | premake=config.get("options","premake") | ||
| 46 : | # make options, passed verbatim after make command | ||
| 47 : | makeopts=config.get("options","makeopts") | ||
| 48 : | |||
| 49 : | # parse manager-related configuration from gkb-manager.cfg | ||
| 50 : | config.readfp(open('gkb-manager.cfg')) | ||
| 51 : | msite=config.get('options','msite') | ||
| 52 : | |||
| 53 : | # these values will likely stay fairly static across different builds | ||
| 54 : | patchdir="%s/patches" % buildroot # full path to patch file dir | ||
| 55 : | configdir="%s/configs" % buildroot # path to where configs are to be stored | ||
| 56 : | logdir="%s/logs" % buildroot # the logs will go here | ||
| 57 : | bindir="%s/bin" % buildroot # the directory that binaries built will be sent for pkging | ||
| 58 : | workdir="%s/work" % buildroot # work directory, where builds take place | ||
| 59 : | |||
| 60 : | # setup the global dicts to hold build data, imported from xml | ||
| 61 : | mastertrees={} | ||
| 62 : | builds={} | ||
| 63 : | |||
| 64 : | def printverbose(myoutput): | ||
| 65 : | """Supporting method to output info to stdout if verbosity level is set""" | ||
| 66 : | if verbose==1: | ||
| 67 : | print myoutput | ||
| 68 : | |||
| 69 : | def log(text,kinfo=0): | ||
| 70 : | if kinfo: | ||
| 71 : | printverbose(kinfo["name"] + " : " + text) | ||
| 72 : | else: | ||
| 73 : | printverbose("gkb : " + text) | ||
| 74 : | |||
| 75 : | def deathbyerror(mymsg): | ||
| 76 : | """Supporting method to print error to stdout and return system error -1 on exit""" | ||
| 77 : | print mymsg | ||
| 78 : | sys.exit(-1) | ||
| 79 : | |||
| 80 : | def verifydir(mydir,kinfo=0): | ||
| 81 : | """Supporting method to verify a given exists, if not it will create it""" | ||
| 82 : | log("verifying directory " + mydir,kinfo) | ||
| 83 : | if not os.path.isdir(mydir): | ||
| 84 : | log("%s doesn't exist, creating" % mydir,kinfo) | ||
| 85 : | os.mkdir(mydir) | ||
| 86 : | return mydir | ||
| 87 : | |||
| 88 : | def chdir(mydir,kinfo): | ||
| 89 : | verifydir(mydir,kinfo) | ||
| 90 : | log("entering "+mydir,kinfo) | ||
| 91 : | os.chdir(mydir) | ||
| 92 : | |||
| 93 : | def verifyfile(myfile,kinfo): | ||
| 94 : | """Supporting method to verify file exists, if not it will exit with error""" | ||
| 95 : | if not os.path.isfile(myfile): | ||
| 96 : | deathbyerror("%s : cannot find file expected at %s, exiting" % (kinfo["name"],myfile)) | ||
| 97 : | return myfile | ||
| 98 : | |||
| 99 : | def md5sum(fileobj): | ||
| 100 : | """calculates the md5sum for fileobj and returns it in hexadecimal""" | ||
| 101 : | md = md5.new() | ||
| 102 : | data = fileobj.read(8192) | ||
| 103 : | while (data): | ||
| 104 : | md.update(data) | ||
| 105 : | data = fileobj.read(8192) | ||
| 106 : | |||
| 107 : | digest = md.hexdigest() | ||
| 108 : | return digest | ||
| 109 : | |||
| 110 : | #def krn_ftpput(filename): | ||
| 111 : | # """uploads the file fileobj to the distribution site""" | ||
| 112 : | #ftp = FTP(mftphost) | ||
| 113 : | #er = ftp.login(mftpuser,mftppass) | ||
| 114 : | #er = ftp.storbinary("STOR "+os.path.basename(filename),open(filename)) | ||
| 115 : | #er = ftp.quit() | ||
| 116 : | |||
| 117 : | def krn_querymgr(command,kinfo): | ||
| 118 : | """queries the build host manager with a variety of commands, such as checkout, checkin, etc.""" | ||
| 119 : | version=krn_localversion(kinfo) | ||
| 120 : | log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo) | ||
| 121 : | result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s\"" % (msite,command,host,passwd,kinfo["name"],version)) | ||
| 122 : | log("result: '"+result+"'",kinfo) | ||
| 123 : | return result=="1" | ||
| 124 : | |||
| 125 : | def krn_localversion(kinfo): | ||
| 126 : | """Checks the version of the local source tree specified in kinfo""" | ||
| 127 : | version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """) | ||
| 128 : | return version | ||
| 129 : | |||
| 130 : | def gkb_build(root,kinfo): | ||
| 131 : | """performs actual kernel builds given root, kinfo (tuple containing relevant data) , premake and makeopts""" | ||
| 132 : | |||
| 133 : | # make sure needed directoriess (for this build) exist, if not create them | ||
| 134 : | |||
| 135 : | verifydir("%s/%s" % (logdir, kinfo["name"]),kinfo) | ||
| 136 : | verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo) | ||
| 137 : | |||
| 138 : | # sync the source to make sure we are up to date ... | ||
| 139 : | log("calling gkb_getsource",kinfo) | ||
| 140 : | gkb_getsource(kinfo) | ||
| 141 : | |||
| 142 : | # now we'll go into the work | ||
| 143 : | chdir(kinfo["workdir"],kinfo) | ||
| 144 : | |||
| 145 : | # check for patches and apply if necessary | ||
| 146 : | gkb_patch(kinfo) | ||
| 147 : | |||
| 148 : | if kinfo["type"] == "kernel24": | ||
| 149 : | krn_build24(kinfo) | ||
| 150 : | elif kinfo["type"] == "kernel26": | ||
| 151 : | krn_build24(kinfo) #run kernel24 for now, change later | ||
| 152 : | |||
| 153 : | # Gerk comment: | ||
| 154 : | # Do we need to change this with 24/26? we probably don't ... I'd rather see us define the list of make targets | ||
| 155 : | # i.e. "dep","clean","vmlinux","modules","modules_install" or "clean","all","modules_install" for 2.6 | ||
| 156 : | # also of note, make all is not what we will always want with 2.6 kernels ... i.e. zImage.prep, zImage.chrp, etc | ||
| 157 : | # also for backward compat they will continue to support calling things individual .. the makefile just does | ||
| 158 : | # this for us with 'all' | ||
| 159 : | |||
| 160 : | |||
| 161 : | def krn_build24(kinfo): | ||
| 162 : | """build a kernel, version 2.4.x""" | ||
| 163 : | |||
| 164 : | if krn_querymgr("checkout",kinfo): | ||
| 165 : | myversion=krn_localversion(kinfo) | ||
| 166 : | |||
| 167 : | # fetch and cp the config file to work/.config | ||
| 168 : | krn_config(kinfo) | ||
| 169 : | |||
| 170 : | # **Note** : we set the preprocessing command to premake inline instead | ||
| 171 : | # of globally as it is only needed in this target | ||
| 172 : | gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts) | ||
| 173 : | |||
| 174 : | # give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature | ||
| 175 : | if clean==1: | ||
| 176 : | gkb_runmake("clean", kinfo, premake, makeopts) | ||
| 177 : | |||
| 178 : | gkb_runmake("dep", kinfo, premake, makeopts) | ||
| 179 : | |||
| 180 : | gkb_runmake(kinfo["binname"], kinfo, premake, makeopts) | ||
| 181 : | |||
| 182 : | # We should check to see if binary built ok, if not bail out | ||
| 183 : | mybindir=bindir+"/linux-"+kinfo["name"]+"-"+myversion | ||
| 184 : | verifydir(mybindir,kinfo) | ||
| 185 : | verifydir(mybindir+"/boot",kinfo) | ||
| 186 : | |||
| 187 : | kbinloc=kinfo["workdir"]+"/"+kinfo["binpath"]+"/"+kinfo["binname"] | ||
| 188 : | if verifyfile(kbinloc,kinfo): | ||
| 189 : | # the binary exists, so let's cp it to bin... | ||
| 190 : | copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion) | ||
| 191 : | else: | ||
| 192 : | # the binary is not there, inform user and bail out with error | ||
| 193 : | deathbyerror("%s : %s is not present, assuming build failure and exiting. See log for details." % (kinfo["name"], kbinloc)) | ||
| 194 : | |||
| 195 : | # now that we know he binary built, let's continue | ||
| 196 : | gkb_runmake("modules",kinfo, premake, makeopts) | ||
| 197 : | |||
| 198 : | # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead | ||
| 199 : | # of globally as it is only needed in this target | ||
| 200 : | gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts)) | ||
| 201 : | |||
| 202 : | chdir(bindir,kinfo) | ||
| 203 : | log("compressing binary archive %s-%s" % (kinfo["name"],myversion),kinfo) | ||
| 204 : | if os.system("tar cjf linux-%s-%s.tar.bz2 %s" % (kinfo["name"], myversion, os.path.basename(mybindir))): | ||
| 205 : | deathbyerror("%s : failed to `tar cjf` %s-%s" % (kinfo["name"], kinfo["name"], myversion)) | ||
| 206 : | |||
| 207 : | os.system("rm -rf %s" % mybindir) | ||
| 208 : | |||
| 209 : | krn_querymgr("checkin",kinfo) | ||
| 210 : | |||
| 211 : | def krn_build26(kinfo): | ||
| 212 : | """build a kernel, version 2.6""" | ||
| 213 : | |||
| 214 : | # source get routines | ||
| 215 : | def gkb_getsource(kinfo): | ||
| 216 : | """method to perform source sync on demand, currenty only supports rsync, but others can be added""" | ||
| 217 : | |||
| 218 : | method=mastertrees[kinfo["mastertree"]]["method"] | ||
| 219 : | |||
| 220 : | if method == "rsync": | ||
| 221 : | get_rsync(kinfo) | ||
| 222 : | elif method == "wget": | ||
| 223 : | get_wget(kinfo) | ||
| 224 : | elif method == "vanilla": | ||
| 225 : | get_vanilla(kinfo) | ||
| 226 : | |||
| 227 : | def get_rsync(kinfo): | ||
| 228 : | """sync the source using rsync""" | ||
| 229 : | args=mastertrees[kinfo["mastertree"]]["args"] | ||
| 230 : | |||
| 231 : | if clean==1: | ||
| 232 : | syncoptions="rsync -azv --delete" | ||
| 233 : | else: | ||
| 234 : | syncoptions="rsync -azv" | ||
| 235 : | |||
| 236 : | syncline=args+" "+kinfo["workdir"] | ||
| 237 : | |||
| 238 : | log("running rsync : %s %s" % (syncoptions, syncline),kinfo) | ||
| 239 : | if os.system("%s %s > %s/%s/rsync.log 2>&1" % (syncoptions, syncline, logdir, kinfo["name"])): | ||
| 240 : | deathbyerror("%s : sync failed, tried %s. See log for details." % (kinfo["name"], syncline) ) | ||
| 241 : | |||
| 242 : | # Gerk comment: | ||
| 243 : | # This above might cause problems ... it (I think) relies on anything coming from stderr to tell | ||
| 244 : | # if it has in fact died... we should test this out. If this is the case me might need to make | ||
| 245 : | # a stub of some sorts to handle the build/error return process... | ||
| 246 : | |||
| 247 : | |||
| 248 : | def get_wget(kinfo): | ||
| 249 : | #needs work | ||
| 250 : | """sync the source using wget and a tar.bz2""" | ||
| 251 : | args=mastertrees[kinfo["mastertree"]]["args"] | ||
| 252 : | |||
| 253 : | log("fetching source archive " % args,kinfo) | ||
| 254 : | mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"]) | ||
| 255 : | |||
| 256 : | if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])): | ||
| 257 : | deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"])) | ||
| 258 : | |||
| 259 : | log("decompressing source file",kinfo) | ||
| 260 : | if os.system("tar xjf " % (myconfigfile,msite,kinfo["name"])): | ||
| 261 : | deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"])) | ||
| 262 : | |||
| 263 : | def get_vanilla(kinfo): | ||
| 264 : | args=mastertrees[kinfo["mastertree"]]["args"] | ||
| 265 : | |||
| 266 : | log("fetching source archive %s" % args) | ||
| 267 : | mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"]) | ||
| 268 : | |||
| 269 : | if os.system("wget -c --output-document=%s %s" % (mysourcefile,args)): | ||
| 270 : | deathbyerror("%s : unable to download source file %s, aborting." % (kinfo["name"], args)) | ||
| 271 : | |||
| 272 : | log("decompressing source file %s" % mysourcefile,kinfo) | ||
| 273 : | |||
| 274 : | #save the current working ectory | ||
| 275 : | cwd=getoutput("pwd") | ||
| 276 : | |||
| 277 : | chdir(workdir,kinfo) | ||
| 278 : | pfd=os.popen("tar vxjf %s" % mysourcefile,"r") | ||
| 279 : | trash=data=pfd.read(255) | ||
| 280 : | |||
| 281 : | while trash: | ||
| 282 : | trash=pfd.read(4096) | ||
| 283 : | |||
| 284 : | if pfd.close(): | ||
| 285 : | deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"])) | ||
| 286 : | |||
| 287 : | fnames=split(data) | ||
| 288 : | name=fnames[0] | ||
| 289 : | |||
| 290 : | os.system("rm -rf %s" % kinfo["workdir"]) | ||
| 291 : | move(work + "/" + dirname, kinfo["workdir"]) | ||
| 292 : | |||
| 293 : | #restore the previous working ectory | ||
| 294 : | chdir(cwd,kinfo) | ||
| 295 : | |||
| 296 : | def gkb_patch(kinfo): | ||
| 297 : | """method that applies patch found in kinfo config files for running kernel build""" | ||
| 298 : | if kinfo["patches"]: | ||
| 299 : | patches=split(kinfo["patches"],";") | ||
| 300 : | |||
| 301 : | for patch in patches: | ||
| 302 : | log("downloading patch %s" % patch,kinfo) | ||
| 303 : | |||
| 304 : | #set a name for this patch file | ||
| 305 : | mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch) | ||
| 306 : | |||
| 307 : | # download pathfile or bail | ||
| 308 : | if os.system("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)): | ||
| 309 : | deathbyerror("%s : unable to download patchfile %s, aborting." % (kinfo["name"],patch)) | ||
| 310 : | |||
| 311 : | # we have a patch file, apply it or bail | ||
| 312 : | log("perfoming patch with %s" % patch,kinfo) | ||
| 313 : | patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch) | ||
| 314 : | log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo) | ||
| 315 : | chdir(kinfo["workdir"],kinfo) | ||
| 316 : | if os.system(patchcommand): | ||
| 317 : | deathbyerror("%s : patchfile %s failed, aborting. See patch log for details." % (kinfo["name"],patch)) | ||
| 318 : | else: | ||
| 319 : | log("no patchfiles, continuing",kinfo) | ||
| 320 : | |||
| 321 : | def krn_config(kinfo): | ||
| 322 : | """method to fetch and place config file for running kernel build""" | ||
| 323 : | |||
| 324 : | if kinfo["config"]==1: | ||
| 325 : | log("fetching config file",kinfo) | ||
| 326 : | myconfigfile="%s/%s.config" % (configdir,kinfo["name"]) | ||
| 327 : | |||
| 328 : | if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])): | ||
| 329 : | deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"])) | ||
| 330 : | |||
| 331 : | log("copying config file to %s/.config" % kinfo["workdir"],kinfo) | ||
| 332 : | copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"]) | ||
| 333 : | |||
| 334 : | # Methods to support build() | ||
| 335 : | def gkb_runmake(command, kinfo, premake, makeopts): | ||
| 336 : | """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)""" | ||
| 337 : | log("running make %s" % command,kinfo) | ||
| 338 : | if os.system("%s make %s %s > %s/%s/make-%s.log 2>&1" % (premake, makeopts, command, logdir, kinfo["name"], command)): | ||
| 339 : | deathbyerror("%s : unable to run make %s, aborting." % (kinfo["name"],command)) | ||
| 340 : | # Gerk comment: | ||
| 341 : | # see other comment above about shunting 2>log | ||
| 342 : | |||
| 343 : | def gkb_parsexml(name): | ||
| 344 : | """parse the xml build file into mastertrees and builds""" | ||
| 345 : | def start_element(name, attrs): | ||
| 346 : | #print "In start_element: %s %s" % (name, attrs) | ||
| 347 : | if name=="mastertree": | ||
| 348 : | mastertrees[attrs["name"]]=attrs | ||
| 349 : | elif name=="build": | ||
| 350 : | builds[attrs["name"]]=attrs | ||
| 351 : | |||
| 352 : | p = xml.parsers.expat.ParserCreate() | ||
| 353 : | p.StartElementHandler = start_element | ||
| 354 : | p.ParseFile(open(name)) | ||
| 355 : | |||
| 356 : | def main(): | ||
| 357 : | """ main program gets executed here """ | ||
| 358 : | |||
| 359 : | # get our working ectory | ||
| 360 : | root=getoutput("pwd") | ||
| 361 : | |||
| 362 : | today=getoutput("date +%D") | ||
| 363 : | buildtime=getoutput("date +'%R:%S %Z'") | ||
| 364 : | |||
| 365 : | print "GKB started %s %s" % (today,buildtime) | ||
| 366 : | |||
| 367 : | # verify the existence important directories, and create if necessary | ||
| 368 : | verifydir(logdir) | ||
| 369 : | verifydir(configdir) | ||
| 370 : | verifydir(patchdir) | ||
| 371 : | verifydir(bindir) | ||
| 372 : | verifydir("%s/work" % buildroot) # build dir, make sure it exists | ||
| 373 : | |||
| 374 : | # download the build jobs from the master site | ||
| 375 : | if os.system("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)): | ||
| 376 : | deathbyerror("Unable to download build config from master site %s, aborting." % msite) | ||
| 377 : | |||
| 378 : | # sets up 'mastertrees' and 'builds' dicts | ||
| 379 : | gkb_parsexml('gkb.xml') | ||
| 380 : | |||
| 381 : | for bdict in builds.values(): | ||
| 382 : | myworkdir="%s/%s" % (workdir,bdict["mastertree"]) | ||
| 383 : | bdict["workdir"]=myworkdir | ||
| 384 : | |||
| 385 : | # for now just call the build | ||
| 386 : | gkb_build(root, bdict) | ||
| 387 : | |||
| 388 : | endtime=getoutput("date +'%R:%S %Z'") | ||
| 389 : | |||
| 390 : | print "GKB finished %s %s" % (today,endtime) | ||
| 391 : | |||
| 392 : | # and finally, call the mainloop to execute | ||
| 393 : | main() | ||
| 394 : |
|
Tobias McNulty Powered by ViewCVS 1.0-dev |
ViewCVS and CVS Help |