#!/usr/bin/python
#
# GKB - GNU Kernel Builder
# Copyright (C) 2003-2004 Tobias McNulty, Mark Guertin
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Ported from PHP version Copyright (c) Tobias McNulty 2000-2003

import sys, os, ConfigParser, popen2
import xml.parsers.expat 
import md5, string
from ftplib import FTP
from commands import getoutput
from shutil import copy, move
from string import split
from urllib2 import urlopen
from ClientForm import ParseResponse


debug=0
clean=1
dont_build=0
verbose=1

# parse host-specific configuration information from gkb.cfg
config = ConfigParser.ConfigParser()
config.readfp(open('gkb-host.cfg'))
buildroot=config.get("options","buildroot")
host=config.get("options","host")
passwd=config.get("options","passwd")

# this one is passed verbatim before the make command.  might be useful for alternate toolchains, etc
# example: premake="HOST=powerpc-unknown-linux-gnu"
premake=config.get("options","premake")
# make options, passed verbatim after make command
makeopts=config.get("options","makeopts")

# parse manager-related configuration from gkb-manager.cfg
config.readfp(open('gkb-manager.cfg'))
msite=config.get('options','msite')

# these values will likely stay fairly static across  different builds
patchdir="%s/patches" % buildroot  # full path to patch file dir
configdir="%s/configs" % buildroot	# path to where configs are to be stored
logdir="%s/logs" % buildroot	# the logs will go here
bindir="%s/bin" % buildroot		# the directory that binaries built will be sent for pkging
workdir="%s/work" % buildroot # work directory, where builds take place

# setup the global dicts to hold build data, imported from xml
mastertrees={}
builds={}

class Error(Exception):
	"""Base class for exceptions in this module."""
	pass

class BuildError(Error):
	"""Exception raised for build-level errors.

	Attributes:
		message -- explanation of the error
	"""

	def __init__(self, message):
		self.message = message

class FatalError(Error):
	"""Exception for fatal errors that prevent program continuation.

	Attributes:
		message -- explanation of why the specific transition is not allowed
	"""

	def __init__(self, message):
		self.message = message

def printverbose(myoutput):
	"""Supporting method to output info to stdout if verbosity level is set"""
	if verbose==1:
		print myoutput

def cleantext(text):
	"""clean a string of passwords, etc. to make it suitable for logging"""
	result = string.replace(text,passwd,"XXXXXXXX")
	return result
	
def linkpipes(input, output):
	"""link two pipes together, writing all the data from input to output"""
	data = input.read(2048)
	while data:
		output.write(data)
		data = input.read(2048)
		
def runcmd(cmd, kinfo=None, outfile=None, append=False, infile=None):
	"""run the specified command, with the optional input and output documents"""
	log("  \_ running " + cleantext(cmd), kinfo)

	p4obj = popen2.Popen4(cmd)
	
	pin = p4obj.tochild
	pouterr = p4obj.fromchild
	
	if infile != None:
		inobj = open(infile, "r")
		linkpipes(inobj, pin)
		inobj.close()
		
	if outfile != None:
		if append:
			mode = "a"
		else:
			mode = "w"
		
		outobj = open(outfile, mode)
		linkpipes(pouterr,outobj)
		outobj.close()
	
	err = p4obj.wait()
	
	if err == None:
		err = 0

	log("    \_ exit code: %i" %  err, kinfo)
	
	return err

def log(text,kinfo=0):
	if kinfo:
		printverbose(kinfo["name"] + " : " + text)
	else:
		printverbose("gkb : " + text)

def verifydir(mydir,kinfo=0):
	"""Supporting method to verify a given  exists, if not it will create it"""
	log("verifying directory " + mydir,kinfo)
	if not os.path.isdir(mydir):
		log("  \_ %s doesn't exist, creating" % mydir,kinfo)
		os.mkdir(mydir)
	return mydir

def chdir(mydir,kinfo):
	verifydir(mydir,kinfo)
	log("entering "+mydir,kinfo)
	os.chdir(mydir)
	
def verifyfile(myfile,kinfo):
	"""Supporting method to verify file exists, if not it will exit with error"""
	if not os.path.isfile(myfile):
		raise BuildError("cannot find file expected at %s, exiting" % myfile)
	return myfile
	
def md5sum(fileobj):
	"""calculates the md5sum for fileobj and returns it in hexadecimal"""
	md = md5.new()
	data = fileobj.read(8192)
	while (data):
		md.update(data)
		data = fileobj.read(8192)
	
	digest = md.hexdigest()
	return digest

def krn_querymgr(command,kinfo,extra=""):
	"""queries the build host manager with a variety of commands, such as checkout, checkin, etc."""
	version=krn_localversion(kinfo)
	log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo)
	result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s%s\"" % (msite,command,host,passwd,kinfo["name"],version,extra))
	log("  \_ result: '"+result+"'",kinfo)
	return result=="1"

def krn_modulesenabled(kinfo):
	"""Checks the source to see if modules are enabled"""
	output=getoutput("""awk -F '=' '/^CONFIG_MODULES/{v=$2} END { printf("%s\\n", v) }' """ + kinfo["workdir"] + """/.config | sed "s/ //g" """)
	if (output=='y'):
		result=1
	else:
		result=0

	return result
	
def krn_localversion(kinfo):
	"""Checks the version of the local source tree specified in kinfo"""
	version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """)
	return version

def gkb_build(root,kinfo):
	"""performs actual kernel builds given root, kinfo (tuple containing relevant data) , premake and makeopts"""
	
	# make sure needed directoriess (for this build) exist, if not create them
	
	verifydir("%s/%s" % (logdir, kinfo["name"]),kinfo)
	verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo)

	# sync the source to make sure we are up to date ...
	log("fetching latest source",kinfo)
	gkb_getsource(kinfo)
	
	# go into the work directory
	#chdir(workdir, kinfo)
	
	# archive the clean source for later uploading
	#log("archiving source to " + kinfo["mastertree"] + ".tar.bz2", kinfo)
	#runcmd("tar cjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"], kinfo)
	
	# now we'll go into the tree's work dir
	chdir(kinfo["workdir"],kinfo)
	
	# check for patches and apply if necessary
	gkb_patch(kinfo)

	if kinfo["type"] == "kernel24":
		krn_build24(kinfo)
	elif kinfo["type"] == "kernel26":
		krn_build24(kinfo) #run kernel24 for now, change later
		
		# Gerk comment:
		# Do we need to change this with 24/26?  we probably don't ... I'd rather see us define the list of make targets
		# i.e. "dep","clean","vmlinux","modules","modules_install" or "clean","all","modules_install" for 2.6
		# also of note, make all is not what we will always want with 2.6 kernels ... i.e. zImage.prep, zImage.chrp, etc
		# also for backward compat they will continue to support calling things individual .. the makefile just does
		# this for us with 'all'
		

def krn_build24(kinfo):
	"""build a kernel, version 2.4.x"""
	
	if krn_querymgr("checkout",kinfo):
		try:
			myversion=krn_localversion(kinfo)
	
			# fetch and cp the config file to work/.config
			krn_config(kinfo)
	
			# **Note** : we set the preprocessing command to premake inline instead
			# of globally as it is only needed in this target
			if dont_build==0:
				gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts)
			
			# give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature
			if (clean==1 and dont_build==0):
				gkb_runmake("clean", kinfo, premake, makeopts)
			
			if dont_build==0:
				gkb_runmake("dep", kinfo, premake, makeopts)
				gkb_runmake(kinfo["binname"], kinfo, premake, makeopts)
			
			# We should check to see if binary built ok, if not bail out
			mybindir=bindir+"/linux-"+kinfo["name"]+"-"+myversion
			verifydir(mybindir,kinfo)
			verifydir(mybindir+"/boot",kinfo)
			
			kbinloc=kinfo["workdir"]+"/"+kinfo["binpath"]+"/"+kinfo["binname"]
			if verifyfile(kbinloc,kinfo):
				# the binary exists, so let's cp it to bin...
				copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)
			else:
				# the binary is not there, inform user and bail out with error
				raise BuildError("%s is not present, assuming build failure and exiting.  See log for details." % kbinloc)
	
			if krn_modulesenabled(kinfo):
				# now that we know he binary built, let's continue
				if dont_build==0:
					gkb_runmake("modules",kinfo, premake, makeopts)
			
				# **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead
				# of globally as it is only needed in this target
				gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))
			else:
				log("skipping make modules (disabled in .config)",kinfo)

			# compress and upload source archive
			chdir(kinfo["workdir"]+"/..",kinfo)
			archive_name = "src-%s.tar.bz2" % kinfo["mastertree"],
			
			# compress and upload kernel binary
			chdir(bindir,kinfo)
			archive_name = "linux-%s-%s.tar.bz2" % (kinfo["name"], myversion)
			
			log("compressing binary archive "+archive_name,kinfo)
			
			if runcmd("tar cjf "+archive_name+" "+os.path.basename(mybindir), kinfo):
				raise BuildError("failed to `tar cjf %s`" %  archive_name)
	
			runcmd("rm -rf %s" % mybindir, kinfo)
			
			krn_upload(archive_name,"kernel",myversion,kinfo)
			krn_querymgr("checkin",kinfo)
			
			if os.fork() == 0:
				#in child
				try:
					# sync the source to make sure we are up to date ...
					log("re-fetching latest source",kinfo)
					gkb_getsource(kinfo)
					
					# go into the work directory
					chdir(workdir, kinfo)
					
					# archive the clean source for later uploading
					log("archiving source to " + kinfo["mastertree"] + ".tar.bz2", kinfo)
					runcmd("tar cjf " + kinfo["mastertree"] + ".tar.bz2 " + kinfo["mastertree"], kinfo)
					
					krn_upload(workdir + "/" + kinfo["mastertree"] + ".tar.bz2", "source", myversion, kinfo)
				finally:
					sys.exit(0)
			
		except BuildError, e:
			log(e.message, kinfo)
			krn_querymgr("checkin",kinfo,"&failed=%s" % krn_localversion(kinfo))
		except:
			krn_querymgr("checkin",kinfo,"&failed=%s" % krn_localversion(kinfo))
			raise
			
def krn_build26(kinfo):
	"""build a kernel, version 2.6"""
	
# source get routines
def gkb_getsource(kinfo):
	"""method to perform source sync on demand, currenty only supports rsync, but others can be added"""
	
	method=mastertrees[kinfo["mastertree"]]["method"]
	
	if method == "rsync":
		get_rsync(kinfo)
	elif method == "wget":
		get_wget(kinfo)
	elif method == "vanilla":
		get_vanilla(kinfo)

def get_rsync(kinfo):
	"""sync the source using rsync"""
	args=mastertrees[kinfo["mastertree"]]["args"]
	
	if clean==1:
		syncoptions="rsync -azv --delete"
	else:
		syncoptions="rsync -azv"
		
	syncline=args+" "+kinfo["workdir"]

	#logging now in runcmd
	#log("running rsync : %s %s" % (syncoptions, syncline),kinfo)
	if runcmd("%s %s" % (syncoptions, syncline), kinfo, "%s/%s/rsync.log" % (logdir, kinfo["name"])):
		raise BuildError("sync failed, tried %s.  See log for details." %  syncline)	
	
def get_wget(kinfo):
	#needs work
	"""sync the source using wget and a tar.bz2"""
	args=mastertrees[kinfo["mastertree"]]["args"]
	
	log("fetching source archive " % args,kinfo)
	mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])
	
	if runcmd("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"]), kinfo):
		raise BuildError("unable to download configfile, aborting.")

	log("decompressing source file",kinfo)
	if runcmd("tar xjf " % (myconfigfile,msite,kinfo["name"]), kinfo):
		raise BuildError("unable to decompress source file, aborting.")

def get_vanilla(kinfo):
	args=mastertrees[kinfo["mastertree"]]["args"]
	
	log("fetching source archive %s" % args)
	mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])
	
	if runcmd("wget -c --output-document=%s %s" % (mysourcefile,args), kinfo):
		raise BuildError("unable to download source file %s, aborting." %  args)

	log("decompressing source file %s" % mysourcefile,kinfo)
	
	#save the current working ectory
	cwd=getoutput("pwd")
	
	chdir(workdir,kinfo)
	pfd=os.popen("tar vxjf %s" % mysourcefile,"r")
	trash=data=pfd.read(255)
	
	while trash:
		trash=pfd.read(4096)

	if pfd.close():
		raise BuildError("unable to decompress source file, aborting.")
	
	fnames=split(data)
	name=fnames[0]
	
	runcmd("rm -rf %s" % kinfo["workdir"], kinfo)
	move(work + "/" + dirname, kinfo["workdir"])
	
	#restore the previous working ectory
	chdir(cwd,kinfo)

def gkb_patch(kinfo):
	"""method that applies patch found in kinfo config files for running kernel build"""
	if kinfo["patches"]:
		patches=split(kinfo["patches"],";")
		
		for patch in patches:
			log("downloading patch %s" % patch,kinfo)
	
			#set a name for this patch file
			mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)
	
			# download pathfile or bail
			if runcmd("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch), kinfo):
				raise BuildError("unable to download patchfile %s, aborting." % patch)
	
			# we have a patch file, apply it or bail
			log("perfoming patch with %s" % patch,kinfo)
			
			# former patchcommand was:
			#patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch)
			#log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)
			
			chdir(kinfo["workdir"],kinfo)
			if runcmd("patch -p1", kinfo, "%s/%s/patch-%s.log" % (logdir,kinfo["name"],patch), False, mypatchfile):
				raise BuildError("patchfile %s failed, aborting.  See patch log for details." % patch)
	else:
		log("no patchfiles, continuing",kinfo)

def krn_config(kinfo):
	"""method to fetch and place config file for running kernel build"""
	
	if kinfo["config"]==1:
		log("fetching config file",kinfo)
		myconfigfile="%s/%s.config" % (configdir,kinfo["name"])
		
		if runcmd("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"]), kinfo):
			raise BuildError("unable to download configfile, aborting.")
	
		log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
		copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])

# Methods to support build()
def gkb_runmake(command, kinfo, premake, makeopts):
	"""Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""
	log("running make %s" % command,kinfo)
	
	# check to see if we need to run a simple make
	# for some reason 2.2 kernels can't handle premake, makeopts, etc.
	localversion = krn_localversion(kinfo)
	
	if (string.find(localversion,"2.2") == 0):
		if runcmd("%s make %s" % (premake,command), kinfo, "%s/%s/make-%s.log" % (logdir, kinfo["name"], command)):
		        raise BuildError("unable to run make %s, aborting." % command)
	else:
		if runcmd("%s make %s %s" % (premake, makeopts, command), kinfo, "%s/%s/make-%s.log" % (logdir, kinfo["name"], command)):
			raise BuildError("unable to run make %s, aborting." % command)
	
def gkb_parsexml(name):
	"""parse the xml build file into mastertrees and builds"""
	def start_element(name, attrs):
		#print "In start_element: %s %s" % (name, attrs)
		if name=="mastertree":
			mastertrees[attrs["name"]]=attrs
		elif name=="build":
			builds[attrs["name"]]=attrs
	
	p = xml.parsers.expat.ParserCreate()
	p.StartElementHandler = start_element
	p.ParseFile(open(name))

def krn_upload(file, type, version, kinfo):
	"""upload the indicated file to the distribution site (kernel archives)"""
	
	if type=="kernel":
		log("uploading kernel version "+version+" to "+msite,kinfo)
	elif type=="source":
		log("uploading source "+kinfo["mastertree"]+" to "+msite,kinfo)
	else:
		raise BuildException, "invalid file upload type: " + type
	
	forms = ParseResponse(urlopen(msite+"/upload.html"))
	form = forms[0]
	
	form["host"] = host
	form["pass"] = passwd
	form["type"] = type
	form["tree"] = kinfo["mastertree"]
	form["build"] = kinfo["name"]
	form["version"] = version
	
	form.add_file(open(file), "application/x-bzip2", os.path.basename(file))

	# form.click() returns a urllib2.Request object
	# (see HTMLForm.click.__doc__ if you don't have urllib2)
	response = urlopen(form.click("cmd"))

	if debug:
		print response.geturl()
		print response.info()  # headers
 		print response.read()  # body
 		
 	log("  \_ response: " + response.read(),kinfo)
 	
 	response.close()

def main():
	""" main program gets executed here """

	# get our working ectory
	root=getoutput("pwd")
	
	today=getoutput("date +%D")
	buildtime=getoutput("date +'%R:%S %Z'")

	print "GNU Kernel Builder started %s %s" % (today,buildtime)
	
	# verify the existence important directories, and create if necessary
	verifydir(logdir)
	verifydir(configdir)
	verifydir(patchdir)
	verifydir(bindir)
	verifydir("%s/work" % buildroot) # build dir, make sure it exists
	
	# download the build jobs from the master site
	log("fetching build jobs from master site")
	if runcmd("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):
		raise FatalError, "Unable to download build jobs from master site %s, aborting." % msite
	
	# sets up 'mastertrees' and 'builds' dicts
	gkb_parsexml('gkb.xml')
	
	for bdict in builds.values():
		myworkdir="%s/%s" % (workdir,bdict["mastertree"])
		bdict["workdir"]=myworkdir
		
		try:
			# for now just call the build
			gkb_build(root, bdict)
		except BuildError, e:
			log(e.message, bdict)
		except KeyboardInterrupt, e:
			log("Caught keyboard interrupt, exiting...")
			break
			
	endtime=getoutput("date +'%R:%S %Z'")

	print "GKB finished %s %s" % (today,endtime)

# and finally, call the mainloop to execute
main()