ViewVC Help
View File | Revision Log | Show Annotations | Root Listing
root/cvsroot/COMP/CRAB/python/DBinterface.py
(Generate patch)

Comparing COMP/CRAB/python/DBinterface.py (file contents):
Revision 1.12 by spiga, Mon Mar 31 09:23:55 2008 UTC vs.
Revision 1.72 by lolass, Mon Jan 21 11:22:46 2013 UTC

# Line 1 | Line 1
1 from crab_logger import Logger
1   from crab_exceptions import *
2   from crab_util import *
3   import common
4   import os, time, shutil
5 + import traceback
6  
7   from ProdCommon.BossLite.API.BossLiteAPI import BossLiteAPI
8 <
8 > from ProdCommon.BossLite.Common.Exceptions import DbError
9 > from ProdCommon.BossLite.Common.Exceptions import TaskError
10  
11   from ProdCommon.BossLite.DbObjects.Job import Job
12   from ProdCommon.BossLite.DbObjects.Task import Task
# Line 26 | Line 27 | class DBinterface:
27          dbname = common.work_space.shareDir()+'crabDB'
28          dbConfig = {'dbName':dbname
29              }
30 +        try:
31 +            common.bossSession = BossLiteAPI( self.db_type, dbConfig)
32 +        except Exception, e :
33 +            raise CrabException('Istantiate DB Session : '+str(e))
34 +
35 +        try:
36 +            common.bossSession.bossLiteDB.installDB('$CRABPRODCOMMONPYTHON/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql')
37 +        except Exception, e :
38 +            raise CrabException('DB Installation error : '+str(e))
39 +        return
40  
30        common.bossSession = BossLiteAPI( self.db_type, dbConfig)
31        common.bossSession.installDB('$CRABPRODCOMMONPYTHON/ProdCommon/BossLite/DbObjects/setupDatabase-sqlite.sql')    
32        
33        return
34
41      def loadDB(self):
42  
43          dbname = common.work_space.shareDir()+'crabDB'
44          dbConfig = {'dbName':dbname
45              }
46 <        common.bossSession = BossLiteAPI( self.db_type, dbConfig)
47 <        self.task = common.bossSession.load(1)[0]
46 >        try:
47 >            common.bossSession = BossLiteAPI( self.db_type, dbConfig)
48 >        except Exception, e :
49 >            raise CrabException('Istantiate DB Session : '+str(e))
50 >
51          return
52  
53 <    def getTask(self, jobsList='all'): #, cfg_params):
53 >    def getTask(self, jobsList='all'):
54          """
55          Return task with all/list of jobs
56          """
57 <
58 <        task = common.bossSession.load(1,jobsList)[0]
57 >        try:
58 >            task = common.bossSession.load(1,jobsList)
59 >        except Exception, e :
60 >            common.logger.debug( "Error while getting task : " +str(traceback.format_exc()))
61 >            raise CrabException('Error while getting task '+str(e))
62          return task
63  
64      def getJob(self, n):
65          """
66          Return a task with a single job
67          """
68 <        task = common.bossSession.load(1,str(n))[0]
68 >        try:
69 >            task = common.bossSession.load(1,str(n))
70 >        except Exception, e :
71 >            common.logger.debug( "Error while getting job : " +str(traceback.format_exc()))
72 >            raise CrabException('Error while getting job '+str(e))
73          return task
74  
75  
# Line 63 | Line 79 | class DBinterface:
79          with the first coniguration stuff
80          """
81          opt={}
82 <        opt['serverName']=optsToSave['server_name']
83 <        opt[ 'name']=common.work_space.taskName()  
84 <        task = Task( opt )
85 <      
86 <        common.bossSession.saveTask( task )
82 >        if optsToSave.get('server_mode',0) == 1: opt['serverName']=optsToSave['server_name']
83 >        if common.scheduler.name().upper() not in ['LSF', 'CAF', 'SGE', 'PBS']:
84 >            checkNewSiteDB()
85 >        opt['name']= getUserName()+ '_' + string.split(common.work_space.topDir(),'/')[-2]+'_'+common.work_space.task_uuid()
86 >        task = Task( opt )
87 >        try:
88 >            common.bossSession.saveTask( task )
89 >        except Exception, e :
90 >            raise CrabException('Error creating task '+str(traceback.format_exc()))
91 >            
92          return
93  
94      def updateTask_(self,optsToSave):      
95          """
96          Update task fields  
97          """
98 <        task = common.bossSession.load(1)[0]
99 <        
98 >        task = self.getTask()
99 >  
100          for key in optsToSave.keys():
101              task[key] = optsToSave[key]
102 <        common.bossSession.updateDB( task )
102 >        try:
103 >            common.bossSession.updateDB( task )
104 >        except Exception, e :
105 >            raise CrabException('Error updating task '+str(traceback.format_exc()))
106 >
107          return
108  
109 <    def createJobs_(self, jobsL):
109 >    def createJobs_(self, jobsL, isNew=True):
110          """  
111          Fill crab DB with  the jobs filed
112          """
113 <        task = common.bossSession.loadTask(1)
113 >        task = self.getTask()
114 >
115          jobs = []
116          for id in jobsL:
117              parameters = {}
118 <            parameters['jobId'] =  str(id)
119 <            parameters['name'] = 'job' + str(id)
118 >            parameters['jobId'] = int(id)
119 >            parameters['taskId'] = 1
120 >            parameters['name'] = task['name'] + '_' + 'job' + str(id)
121              job = Job(parameters)
122 <            jobs.append(job)  
123 <        task.addJobs(jobs)
124 <        common.bossSession.updateDB( task )
122 >            jobs.append(job)
123 >            common.bossSession.getRunningInstance(job)
124 >            job.runningJob['status'] = 'C'
125 >        ## added to support second step creation
126 >        ## maybe it is not needed. TO CLARIFY
127 >        if isNew:
128 >            task.addJobs(jobs)
129 >        else:
130 >            task.appendJobs(jobs)
131 >        try:
132 >            common.bossSession.updateDB( task )
133 >        except Exception, e :
134 >            raise CrabException('Error updating task '+str(traceback.format_exc()))
135 >
136          return
137  
138      def updateJob_(self, jobsL, optsToSave):      
139          """
140          Update Job fields  
141          """
142 <        task = common.bossSession.load(1,jobsL)[0]
142 >        task = self.getTask(jobsL)
143          id =0
144          for job in task.jobs:
145              for key in optsToSave[id].keys():
146                  job[key] = optsToSave[id][key]
147              id+=1
148 <        common.bossSession.updateDB( task )
148 >        try:
149 >            common.bossSession.updateDB( task )
150 >        except Exception, e :
151 >            raise CrabException('Error updating task '+str(traceback.format_exc()))
152          return
153  
154      def updateRunJob_(self, jobsL, optsToSave):      
155          """
156          Update Running Job fields  
157          """
158 <        task = common.bossSession.load(1,jobsL)[0]
158 >        task = self.getTask(jobsL)
159 >
160          id=0
161          for job in task.jobs:
162              common.bossSession.getRunningInstance(job)
# Line 126 | Line 168 | class DBinterface:
168  
169      def nJobs(self,list=''):
170          
171 <        task = common.bossSession.load(1)[0]
171 >        task = self.getTask()
172          listId=[]
173          if list == 'list':
174              for job in task.jobs:listId.append(int(job['jobId']))  
# Line 138 | Line 180 | class DBinterface:
180          """
181           List a complete set of infos for a job/range of jobs  
182          """
183 <        task = common.bossSession.load(1)[0]
142 <
143 <        njobs = len(jobs)
144 <        lines=[]
145 <        header=''
146 <     #   ##query the DB asking the right infos for runningJobs  TODO  DS
147 <     #   for job in jobs:
148 <     #       ## here the query over runngJobs  
149 <     #       pass
150 <
183 >        task = self.getTask(jobs)
184  
185 <     #   ##Define Header to show and Pass the query results,
186 <     #   ##  header and format to displayReport()   TODO  DS
187 <     #   if njobs == 1: plural = ''
188 <     #   else:          plural = 's'
189 <     #   header += 'Listing %d job%s:\n' % (njobs, plural)
190 <     #   header += ' :\n' % (---) ## TODO DS
191 <
192 <     #   displayReport(header, lines):
185 >        print "--------------------------"
186 >        for Job in task.jobs:
187 >            print "Id: ",Job['jobId']
188 >            print "Dest: ", Job['dlsDestination']
189 >            print "Output: ", Job['outputFiles']
190 >            print "Args: ",Job['arguments']
191 >            print "Service: ",Job.runningJob['service']
192 >            print "--------------------------"
193          return      
194  
195      def serializeTask(self, tmp_task = None):
196          if tmp_task is None:
197 <            tmp_task = common.bossSession.load(1)[0]
197 >            tmp_task = self.getTask()
198          return common.bossSession.serialize(tmp_task)  
199  
200 <    def queryID(self,server_mode=0):
200 >    def queryID(self,server_mode=0, jid=False):
201          '''
202          Return the taskId if serevr_mode =1
203          Return the joblistId if serevr_mode =0
204          '''    
205          header=''
206          lines=[]
207 <        task = common.bossSession.load(1)[0]
207 >        task = self.getTask()
208          if server_mode == 1:
209 <            header= "Task Id = %-40s " %(task['name'])
210 <        else:
209 >            # init client server params...
210 >            CliServerParams(self)      
211 >            headerTask = "Task Id = %-40s\n" %(task['name'])
212 >            headerTask+=  '--------------------------------------------------------------------------------------------\n'
213 >            displayReport(self,headerTask,lines)
214 >            common.logger.info(showWebMon(self.server_name))
215 >        if (jid ) or (server_mode == 0):
216              for job in task.jobs:
217                  toPrint=''
218                  common.bossSession.getRunningInstance(job)
219 <                toPrint = "%-5s %-50s " % (job['id'],job.runningJob['schedulerId'])
219 >                toPrint = "%-5s %-50s " % (job['jobId'],job.runningJob['schedulerId'])
220                  lines.append(toPrint)
221 <            header+= "%-5s %-50s " % ('Job:','ID' )
222 <        displayReport(self,header,lines)
221 >            header+= "%-5s %-50s\n " % ('Job:','ID' )
222 >            header+=  '--------------------------------------------------------------------------------------------\n'
223 >            displayReport(self,header,lines)
224          return  
225  
226      def queryTask(self,attr):
227          '''
228          Perform a query over a generic task attribute
229          '''
230 <        task = common.bossSession.loadTask(1)
230 >        task = self.getTask()
231          return task[attr]
232  
233      def queryJob(self, attr, jobsL):
# Line 197 | Line 236 | class DBinterface:
236          over a generic job attribute
237          '''
238          lines=[]
239 <        task = common.bossSession.load(1,jobsL)[0]
239 >        task = self.getTask(jobsL)
240          for job in task.jobs:
241 <            lines.append(eval(job[attr]))
241 >            lines.append(job[attr])
242          return lines
243  
244      def queryRunJob(self, attr, jobsL):
# Line 208 | Line 247 | class DBinterface:
247          over a generic job attribute
248          '''
249          lines=[]
250 <        task = common.bossSession.load(1,jobsL)[0]
250 >        task = self.getTask(jobsL)
251          for job in task.jobs:
252              common.bossSession.getRunningInstance(job)
253              lines.append(job.runningJob[attr])
# Line 219 | Line 258 | class DBinterface:
258          Returns the list of distinct value for a given job attributes
259          '''
260          distAttr=[]
261 <        task = common.bossSession.loadJobDist( 1, attr )
262 <        for i in task: distAttr.append(eval(i[attr]))  
261 >        try:
262 >            task = common.bossSession.loadJobDist( 1, attr )
263 >        except Exception, e :
264 >            common.logger.debug( "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
265 >            raise CrabException('Error loading Jobs By distinct Attr '+str(e))
266 >
267 >        for i in task: distAttr.append(i[attr])  
268          return  distAttr
269  
270      def queryDistJob_Attr(self, attr_1, attr_2, list):
# Line 228 | Line 272 | class DBinterface:
272          Returns the list of distinct value for a given job attribute
273          '''
274          distAttr=[]
275 <        task = common.bossSession.loadJobDistAttr( 1, attr_1, attr_2, list )
276 <        for i in task: distAttr.append(eval(i[attr_1]))  
275 >        try:
276 >            task = common.bossSession.loadJobDistAttr( 1, attr_1, attr_2, list )
277 >        except Exception, e :
278 >            common.logger.debug( "Error loading Jobs By distinct Attr : " +str(traceback.format_exc()))
279 >            raise CrabException('Error loading Jobs By distinct Attr '+str(e))
280 >
281 >        for i in task: distAttr.append(i[attr_1])  
282          return  distAttr
283  
284      def queryAttrJob(self, attr, field):
# Line 237 | Line 286 | class DBinterface:
286          Returns the list of jobs matching the given attribute
287          '''
288          matched=[]
289 <        task = common.bossSession.loadJobsByAttr(attr )
289 >        try:
290 >            task = common.bossSession.loadJobsByAttr(attr )
291 >        except Exception, e :
292 >            common.logger.debug( "Error loading Jobs By Attr : " +str(traceback.format_exc()))
293 >            raise CrabException('Error loading Jobs By Attr '+str(e))
294          for i in task:
295              matched.append(i[field])
296          return  matched
# Line 248 | Line 301 | class DBinterface:
301          Returns the list of jobs matching the given attribute
302          '''
303          matched=[]
304 <        task = common.bossSession.loadJobsByRunningAttr(attr)
304 >        try:
305 >            task = common.bossSession.loadJobsByRunningAttr(attr)
306 >        except Exception, e :
307 >            common.logger.debug( "Error loading Jobs By Running Attr : " +str(traceback.format_exc()))
308 >            raise CrabException('Error loading Jobs By Running Attr '+str(e))
309          for i in task:
310 <            matched.append(i[field])
310 >            matched.append(i.runningJob[field])
311          return matched
312 +
313 +    def newRunJobs(self,nj='all'):
314 +        """
315 +        Get new running instances
316 +        """  
317 +        task = self.getTask(nj)
318 +
319 +        for job in task.jobs:
320 +            common.bossSession.getNewRunningInstance(job)
321 +            job.runningJob['status'] = 'C'
322 +            job.runningJob['statusScheduler'] = 'Created'
323 +            job.runningJob['state'] = 'Created'
324 +        common.bossSession.updateDB(task)    
325 +        return        
326 +
327 +    def deserXmlStatus(self, reportList):
328 +
329 +        task = self.getTask()
330 +        if int(self.cfg_params.get('WMBS.automation',0)) == 1:
331 +            if len(reportList) ==0:
332 +                msg = 'You are using CRAB with WMBS the server is still creating your jobs.\n'
333 +                msg += '\tPlease wait...'
334 +                raise CrabException(msg)
335 +            newJobs =  len(reportList) - len(task.jobs)
336 +            if newJobs != 0:
337 +                isNew=True  
338 +                if len(task.jobs):isNew=False
339 +                jobL=[]  
340 +                for i in range(1,newJobs+1):
341 +                    jobL.append(len(task.jobs)+i)
342 +                self.createJobs_(jobL,isNew)
343 +
344 +        for job in task.jobs:
345 +            if not job.runningJob:
346 +                raise CrabException( "Missing running object for job %s"%str(job['jobId']) )
347 +
348 +            id = str(job.runningJob['jobId'])
349 +            rForJ = None
350 +            nj_list= []
351 +            for r in reportList:
352 +                if r.getAttribute('id') in [ id, 'all']:
353 +                    rForJ = r
354 +                    break
355 +
356 +            # check if rForJ is None
357 +            if rForJ is None:
358 +                common.logger.debug( "Missing XML element for job %s, skip update status"%str(id) )
359 +                continue
360 +            
361 +            ## Check the submission number and create new running jobs on the client side
362 +            if rForJ.getAttribute('resubmit') != 'None' and (rForJ.getAttribute('status') not in ['Cleared','Killed','Done','Done (Failed)','Not Submitted', 'Cancelled by user']) :
363 +                if int(job.runningJob['submission']) < int(rForJ.getAttribute('resubmit')) + 1:
364 +                    nj_list.append(id)
365 +            if len(nj_list) > 0: self.newRunJobs(nj_list)
366 +
367 +        task_new = self.getTask()
368 +
369 +        for job in task_new.jobs:
370 +            id = str(job.runningJob['jobId'])
371 +            # TODO linear search, probably it can be optized with binary search
372 +            rForJ = None
373 +            for r in reportList:
374 +                if r.getAttribute('id') in [ id, 'all']:
375 +                    rForJ = r
376 +                    break
377 +                  
378 +            # Data alignment
379 +            if rForJ.getAttribute('status') not in ['Unknown']: # ['Created', 'Unknown']:
380 +                   # update the status  
381 +                common.logger.debug("Updating DB status for job: " + str(id) + " @: " \
382 +                                      + str(rForJ.getAttribute('status')) )
383 +                job.runningJob['statusScheduler'] = str( rForJ.getAttribute('status') )
384 +                if (rForJ.getAttribute('status') == 'Done' or rForJ.getAttribute('status') == 'Done (Failed)')\
385 +                  and rForJ.getAttribute('sched_status') == 'E' :
386 +                    job.runningJob['status'] = 'SD'
387 +                else:
388 +                    job.runningJob['status'] = str( rForJ.getAttribute('sched_status') )
389 +          
390 +                job.runningJob['schedulerId'] = str( rForJ.getAttribute('sched_id') )
391 +
392 +                job.runningJob['destination'] = str( rForJ.getAttribute('site') )
393 +                dest = str(job.runningJob['destination']).split(':')[0]
394 +              
395 +                job.runningJob['applicationReturnCode'] = str( rForJ.getAttribute('exe_exit') )
396 +                exe_exit_code = str(job.runningJob['applicationReturnCode'])
397 +              
398 +                job.runningJob['wrapperReturnCode'] = str( rForJ.getAttribute('job_exit') )
399 +                job_exit_code = str(job.runningJob['wrapperReturnCode'])
400 +
401 +                job['closed'] = str( rForJ.getAttribute('ended') )
402 +
403 +                job.runningJob['state'] = str( rForJ.getAttribute('action') )
404 +          
405 +                # Needed for unique naming of the output.
406 +                job['arguments'] = "%d %s"%(job.runningJob['jobId'], str(rForJ.getAttribute('submission')).strip() )
407 +          
408 +        common.bossSession.updateDB( task_new )
409 +        return
410 +
411 +    # FIXME temporary method to verify what kind of submission to perform towards the server
412 +    def checkIfNeverSubmittedBefore(self):
413 +        for j in self.getTask().jobs:
414 +            if j.runningJob['submission'] > 1 or j.runningJob['state'] != 'Created':
415 +                return False
416 +        return True
417 +
418 +    # Method to update arguments w.r.t. resubmission number in order to grant unique output
419 +    def updateResubAttribs(self, jobsL):
420 +        task = self.getTask(jobsL)
421 +        for j in task.jobs:
422 +            common.bossSession.getRunningInstance(j)
423 +            try:
424 +                resubNum = int(str(j['arguments']).split(' ')[1]) + 1
425 +            except Exception, e:
426 +                resubNum = j.runningJob['submission']
427 +            newArgs = "%d %d"%(j.runningJob['jobId'], resubNum)
428 +            j['arguments'] = newArgs
429 +
430 +        common.bossSession.updateDB(task)
431 +        return
432 +

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines