ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/DBinterface.py

Comparing COMP/CRAB/python/DBinterface.py (file contents):
Revision 1.15 by mcinquil, Tue Apr 15 14:13:07 2008 UTC vs.
Revision 1.66 by farinafa, Tue Dec 15 13:13:40 2009 UTC

#	Line 1 | Line 1
1	–	from crab_logger import Logger
1		from crab_exceptions import *
2		from crab_util import *
3		import common
#	Line 34 | Line 33 | class DBinterface:
33		raise CrabException('Istantiate DB Session : '+str(e))
34
35		try:
36	<	common.bossSession.installDB('$CRABPRODCOMMONPYTHON/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql')
36	>	common.bossSession.bossLiteDB.installDB('$CRABPRODCOMMONPYTHON/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql')
37		except Exception, e :
38		raise CrabException('DB Installation error : '+str(e))
39		return
#	Line 56 | Line 55 | class DBinterface:
55		Return task with all/list of jobs
56		"""
57		try:
58	<	task = common.bossSession.load(1,jobsList)[0]
58	>	task = common.bossSession.load(1,jobsList)
59		except Exception, e :
60	<	common.logger.debug(3, "Error while getting task : " +str(traceback.format_exc()))
60	>	common.logger.debug( "Error while getting task : " +str(traceback.format_exc()))
61		raise CrabException('Error while getting task '+str(e))
62		return task
63
#	Line 67 | Line 66 | class DBinterface:
66		Return a task with a single job
67		"""
68		try:
69	<	task = common.bossSession.load(1,str(n))[0]
69	>	task = common.bossSession.load(1,str(n))
70		except Exception, e :
71	<	common.logger.debug(3, "Error while getting job : " +str(traceback.format_exc()))
71	>	common.logger.debug( "Error while getting job : " +str(traceback.format_exc()))
72		raise CrabException('Error while getting job '+str(e))
73		return task
74
#	Line 81 | Line 80 | class DBinterface:
80		"""
81		opt={}
82		if optsToSave.get('server_mode',0) == 1: opt['serverName']=optsToSave['server_name']
83	<	opt['name']=common.work_space.taskName()
83	>	opt['name']= getUserName()+ '_' + string.split(common.work_space.topDir(),'/')[-2]+'_'+common.work_space.task_uuid()
84		task = Task( opt )
85		try:
86		common.bossSession.saveTask( task )
87		except Exception, e :
89	–	# common.logger.debug(3, "Error creating task : " +str(traceback.format_exc()))
90	–	# raise CrabException('Error creating task '+str(e))
88		raise CrabException('Error creating task '+str(traceback.format_exc()))
89
90		return
#	Line 107 | Line 104 | class DBinterface:
104
105		return
106
107	<	def createJobs_(self, jobsL):
107	>	def createJobs_(self, jobsL, isNew=True):
108		"""
109		Fill crab DB with  the jobs filed
110		"""
#	Line 116 | Line 113 | class DBinterface:
113		jobs = []
114		for id in jobsL:
115		parameters = {}
116	<	parameters['jobId'] =  str(id)
116	>	parameters['jobId'] = int(id)
117	>	parameters['taskId'] = 1
118		parameters['name'] = task['name'] + '_' + 'job' + str(id)
119		job = Job(parameters)
120	<	jobs.append(job)
121	<	task.addJobs(jobs)
120	>	jobs.append(job)
121	>	common.bossSession.getRunningInstance(job)
122	>	job.runningJob['status'] = 'C'
123	>	## added to support second step creation
124	>	## maybe it is not needed. TO CLARIFY
125	>	if isNew:
126	>	task.addJobs(jobs)
127	>	else:
128	>	task.appendJobs(jobs)
129		try:
130		common.bossSession.updateDB( task )
131		except Exception, e :
#	Line 173 | Line 178 | class DBinterface:
178		"""
179		List a complete set of infos for a job/range of jobs
180		"""
181	<	task = self.getTask()
177	<
178	<	njobs = len(jobs)
179	<	lines=[]
180	<	header=''
181	<	#   ##query the DB asking the right infos for runningJobs  TODO  DS
182	<	#   for job in jobs:
183	<	#       ## here the query over runngJobs
184	<	#       pass
185	<
181	>	task = self.getTask(jobs)
182
183	<	#   ##Define Header to show and Pass the query results,
184	<	#   ##  header and format to displayReport()   TODO  DS
185	<	#   if njobs == 1: plural = ''
186	<	#   else:          plural = 's'
187	<	#   header += 'Listing %d job%s:\n' % (njobs, plural)
188	<	#   header += ' :\n' % (---) ## TODO DS
189	<
190	<	#   displayReport(header, lines):
183	>	print "--------------------------"
184	>	for Job in task.jobs:
185	>	print "Id: ",Job['jobId']
186	>	print "Dest: ", Job['dlsDestination']
187	>	print "Output: ", Job['outputFiles']
188	>	print "Args: ",Job['arguments']
189	>	print "Service: ",Job.runningJob['service']
190	>	print "--------------------------"
191		return
192
193		def serializeTask(self, tmp_task = None):
#	Line 199 | Line 195 | class DBinterface:
195		tmp_task = self.getTask()
196		return common.bossSession.serialize(tmp_task)
197
198	<	def queryID(self,server_mode=0):
198	>	def queryID(self,server_mode=0, jid=False):
199		'''
200		Return the taskId if serevr_mode =1
201		Return the joblistId if serevr_mode =0
#	Line 208 | Line 204 | class DBinterface:
204		lines=[]
205		task = self.getTask()
206		if server_mode == 1:
207	<	header= "Task Id = %-40s " %(task['name'])
208	<	else:
207	>	# init client server params...
208	>	CliServerParams(self)
209	>	headerTask = "Task Id = %-40s\n" %(task['name'])
210	>	headerTask+=  '--------------------------------------------------------------------------------------------\n'
211	>	displayReport(self,headerTask,lines)
212	>	common.logger.info(showWebMon(self.server_name))
213	>	if (jid ) or (server_mode == 0):
214		for job in task.jobs:
215		toPrint=''
216		common.bossSession.getRunningInstance(job)
217	<	toPrint = "%-5s %-50s " % (job['id'],job.runningJob['schedulerId'])
217	>	toPrint = "%-5s %-50s " % (job['jobId'],job.runningJob['schedulerId'])
218		lines.append(toPrint)
219	<	header+= "%-5s %-50s " % ('Job:','ID' )
220	<	displayReport(self,header,lines)
219	>	header+= "%-5s %-50s\n " % ('Job:','ID' )
220	>	header+=  '--------------------------------------------------------------------------------------------\n'
221	>	displayReport(self,header,lines)
222		return
223
224		def queryTask(self,attr):
#	Line 234 | Line 236 | class DBinterface:
236		lines=[]
237		task = self.getTask(jobsL)
238		for job in task.jobs:
239	<	lines.append(eval(job[attr]))
239	>	lines.append(job[attr])
240		return lines
241
242		def queryRunJob(self, attr, jobsL):
#	Line 257 | Line 259 | class DBinterface:
259		try:
260		task = common.bossSession.loadJobDist( 1, attr )
261		except Exception, e :
262	<	common.logger.debug(3, "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
262	>	common.logger.debug( "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
263		raise CrabException('Error loading Jobs By distinct Attr '+str(e))
264
265	<	for i in task: distAttr.append(eval(i[attr]))
265	>	for i in task: distAttr.append(i[attr])
266		return  distAttr
267
268		def queryDistJob_Attr(self, attr_1, attr_2, list):
#	Line 271 | Line 273 | class DBinterface:
273		try:
274		task = common.bossSession.loadJobDistAttr( 1, attr_1, attr_2, list )
275		except Exception, e :
276	<	common.logger.debug(3, "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
276	>	common.logger.debug( "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
277		raise CrabException('Error loading Jobs By distinct Attr '+str(e))
278
279	<	for i in task: distAttr.append(eval(i[attr_1]))
279	>	for i in task: distAttr.append(i[attr_1])
280		return  distAttr
281
282		def queryAttrJob(self, attr, field):
#	Line 285 | Line 287 | class DBinterface:
287		try:
288		task = common.bossSession.loadJobsByAttr(attr )
289		except Exception, e :
290	<	common.logger.debug(3, "Error loading Jobs By Attr : " +str(traceback.format_exc()))
290	>	common.logger.debug( "Error loading Jobs By Attr : " +str(traceback.format_exc()))
291		raise CrabException('Error loading Jobs By Attr '+str(e))
292		for i in task:
293		matched.append(i[field])
#	Line 300 | Line 302 | class DBinterface:
302		try:
303		task = common.bossSession.loadJobsByRunningAttr(attr)
304		except Exception, e :
305	<	common.logger.debug(3, "Error loading Jobs By Running Attr : " +str(traceback.format_exc()))
305	>	common.logger.debug( "Error loading Jobs By Running Attr : " +str(traceback.format_exc()))
306		raise CrabException('Error loading Jobs By Running Attr '+str(e))
307		for i in task:
308	<	matched.append(i[field])
308	>	matched.append(i.runningJob[field])
309		return matched
310	+
311	+	def newRunJobs(self,nj='all'):
312	+	"""
313	+	Get new running instances
314	+	"""
315	+	task = self.getTask(nj)
316	+
317	+	for job in task.jobs:
318	+	common.bossSession.getNewRunningInstance(job)
319	+	job.runningJob['status'] = 'C'
320	+	job.runningJob['statusScheduler'] = 'Created'
321	+	job.runningJob['state'] = 'Created'
322	+	common.bossSession.updateDB(task)
323	+	return
324	+
325	+	def deserXmlStatus(self, reportList):
326	+
327	+	task = self.getTask()
328	+	if int(self.cfg_params.get('WMBS.automation',0)) == 1:
329	+	if len(reportList) ==0:
330	+	msg = 'You are using CRAB with WMBS the server is still creating your jobs.\n'
331	+	msg += '\tPlease wait...'
332	+	raise CrabException(msg)
333	+	newJobs =  len(reportList) - len(task.jobs)
334	+	if newJobs != 0:
335	+	isNew=True
336	+	if len(task.jobs):isNew=False
337	+	jobL=[]
338	+	for i in range(1,newJobs+1):
339	+	jobL.append(len(task.jobs)+i)
340	+	self.createJobs_(jobL,isNew)
341	+
342	+	for job in task.jobs:
343	+	if not job.runningJob:
344	+	raise CrabException( "Missing running object for job %s"%str(job['jobId']) )
345	+
346	+	id = str(job.runningJob['jobId'])
347	+	rForJ = None
348	+	nj_list= []
349	+	for r in reportList:
350	+	if r.getAttribute('id') in [ id, 'all']:
351	+	rForJ = r
352	+	break
353	+	## Check the submission number and create new running jobs on the client side
354	+	if rForJ.getAttribute('resubmit') != 'None' and (rForJ.getAttribute('status') not in ['Cleared','Killed','Done','Done (Failed)','Not Submitted', 'Cancelled by user']) :
355	+	if int(job.runningJob['submission']) < int(rForJ.getAttribute('resubmit')) + 1:
356	+	nj_list.append(id)
357	+	if len(nj_list) > 0: self.newRunJobs(nj_list)
358	+
359	+	task_new = self.getTask()
360	+
361	+	for job in task_new.jobs:
362	+	id = str(job.runningJob['jobId'])
363	+	# TODO linear search, probably it can be optized with binary search
364	+	rForJ = None
365	+	for r in reportList:
366	+	if r.getAttribute('id') in [ id, 'all']:
367	+	rForJ = r
368	+	break
369	+
370	+	# Data alignment
371	+	if rForJ.getAttribute('status') not in ['Unknown']: # ['Created', 'Unknown']:
372	+	# update the status
373	+	common.logger.debug("Updating DB status for job: " + str(id) + " @: " \
374	+	+ str(rForJ.getAttribute('status')) )
375	+	job.runningJob['statusScheduler'] = str( rForJ.getAttribute('status') )
376	+	if (rForJ.getAttribute('status') == 'Done' or rForJ.getAttribute('status') == 'Done (Failed)')\
377	+	and rForJ.getAttribute('sched_status') == 'E' :
378	+	job.runningJob['status'] = 'SD'
379	+	else:
380	+	job.runningJob['status'] = str( rForJ.getAttribute('sched_status') )
381	+
382	+	job.runningJob['schedulerId'] = str( rForJ.getAttribute('sched_id') )
383	+
384	+	job.runningJob['destination'] = str( rForJ.getAttribute('site') )
385	+	dest = str(job.runningJob['destination']).split(':')[0]
386	+
387	+	job.runningJob['applicationReturnCode'] = str( rForJ.getAttribute('exe_exit') )
388	+	exe_exit_code = str(job.runningJob['applicationReturnCode'])
389	+
390	+	job.runningJob['wrapperReturnCode'] = str( rForJ.getAttribute('job_exit') )
391	+	job_exit_code = str(job.runningJob['wrapperReturnCode'])
392	+
393	+	job['closed'] = str( rForJ.getAttribute('ended') )
394	+
395	+	job.runningJob['state'] = str( rForJ.getAttribute('action') )
396	+
397	+	#if str( rForJ.getAttribute('resubmit') ).isdigit():
398	+	#    job['submissionNumber'] = int(rForJ.getAttribute('resubmit'))
399	+	#    job.runningJob['submission'] =  int(rForJ.getAttribute('resubmit'))
400	+
401	+	# TODO cleared='0' field, how should it be handled/mapped in BL? #Fabio
402	+
403	+	common.bossSession.updateDB( task_new )
404	+	return
405	+
406	+	# FIXME temporary method to verify what kind of submission to perform towards the server
407	+	def checkIfNeverSubmittedBefore(self):
408	+	for j in self.getTask().jobs:
409	+	if j.runningJob['submission'] > 1 or j.runningJob['state'] != 'Created':
410	+	return False
411	+	return True
412	+
413	+	# Method to update arguments w.r.t. resubmission number in order to grant unique output
414	+	def updateResubAttribs(self, jobsL):
415	+	task = self.getTask(jobsL)
416	+	for j in task.jobs:
417	+	common.bossSession.getRunningInstance(j)
418	+	newArgs = "%d %d"%(j.runningJob['jobId'], j.runningJob['submission'])
419	+	j['arguments'] = newArgs
420	+
421	+	common.bossSession.updateDB(task)
422	+	return
423	+
424	+

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines