| Revision: | 1.75 |
| Committed: | Fri May 29 15:10:50 2009 UTC (15 years, 11 months ago) by spiga |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_6_6_pre6, CRAB_2_7_1_pre2, CRAB_2_6_6_pre5, CRAB_2_7_1_pre1, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_7_0, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, CRAB_2_6_0_pre7, CRAB_2_6_0_pre6, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Changes since 1.74: | +3 -2 lines |
| Error occurred while calculating annotation data. | |
| Log Message: | improved logging |
| # | Content |
|---|---|
| 1 | from Actor import Actor |
| 2 | from WorkSpace import WorkSpace |
| 3 | from JobList import JobList |
| 4 | from ScriptWriter import ScriptWriter |
| 5 | from Scheduler import Scheduler |
| 6 | from crab_exceptions import * |
| 7 | from crab_util import * |
| 8 | import common |
| 9 | |
| 10 | import os, string, math |
| 11 | import time |
| 12 | |
| 13 | class Creator(Actor): |
| 14 | def __init__(self, job_type_name, cfg_params, ncjobs, skip_blocks=False,isNew=True,first_jobID=0): |
| 15 | self.job_type_name = job_type_name |
| 16 | self.job_type = None |
| 17 | self.cfg_params = cfg_params |
| 18 | self.total_njobs = 0 |
| 19 | self.total_number_of_events = 0 |
| 20 | self.job_number_of_events = 0 |
| 21 | self.first_event = 0 |
| 22 | self.jobParamsList=[] |
| 23 | self.first_jobID = first_jobID |
| 24 | self.isNew = isNew |
| 25 | |
| 26 | self.createJobTypeObject(ncjobs,skip_blocks,self.isNew) |
| 27 | common.logger.debug( __name__+": JobType "+self.job_type.name()+" created") |
| 28 | |
| 29 | |
| 30 | self.total_njobs = self.job_type.numberOfJobs(); |
| 31 | common.logger.debug( __name__+": total # of jobs = "+`self.total_njobs`) |
| 32 | |
| 33 | self.ncjobs = ncjobs |
| 34 | if ncjobs == 'all' : self.ncjobs = self.total_njobs |
| 35 | if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs |
| 36 | |
| 37 | self.job_type_name = self.job_type.name() |
| 38 | |
| 39 | common.logger.debug( "Creator constructor finished") |
| 40 | return |
| 41 | |
| 42 | def writeJobsSpecsToDB(self, first_jobID=0): |
| 43 | """ |
| 44 | Write firstEvent and maxEvents in the DB for future use |
| 45 | """ |
| 46 | |
| 47 | self.job_type.split(self.jobParamsList,first_jobID) |
| 48 | return |
| 49 | |
| 50 | def nJobs(self): |
| 51 | return self.total_njobs |
| 52 | |
| 53 | |
| 54 | def nJobsL(self): |
| 55 | jobsL=[] |
| 56 | for i in range(self.total_njobs): |
| 57 | jobsL.append(self.first_jobID+i+1) |
| 58 | return jobsL |
| 59 | |
| 60 | def createJobTypeObject(self,ncjobs,skip_blocks,isNew): |
| 61 | file_name = 'cms_'+ string.lower(self.job_type_name) |
| 62 | klass_name = string.capitalize(self.job_type_name) |
| 63 | |
| 64 | try: |
| 65 | klass = importName(file_name, klass_name) |
| 66 | except KeyError: |
| 67 | msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`' |
| 68 | raise CrabException(msg) |
| 69 | except ImportError, e: |
| 70 | msg = 'Cannot create job type '+self.job_type_name |
| 71 | msg += ' (file: '+file_name+', class '+klass_name+'):\n' |
| 72 | msg += str(e) |
| 73 | raise CrabException(msg) |
| 74 | |
| 75 | self.job_type = klass(self.cfg_params,ncjobs,skip_blocks,isNew) |
| 76 | return |
| 77 | |
| 78 | def jobType(self): |
| 79 | return self.job_type |
| 80 | |
| 81 | def run(self): |
| 82 | """ |
| 83 | The main method of the class. |
| 84 | """ |
| 85 | |
| 86 | common.logger.debug( "Creator::run() called") |
| 87 | start = time.time() |
| 88 | # Instantiate ScriptWriter |
| 89 | script_writer = ScriptWriter(self.cfg_params,'crab_template.sh') |
| 90 | |
| 91 | # Loop over jobs |
| 92 | argsList = [] |
| 93 | njc = 0 |
| 94 | listID=[] |
| 95 | listField=[] |
| 96 | listRunField=[] |
| 97 | run_jobToSave = {'status' :'C', \ |
| 98 | 'statusScheduler' : 'Created', \ |
| 99 | 'state' : "Created" } |
| 100 | |
| 101 | msg = ("Creating jobs :\n") |
| 102 | for nj in range(self.total_njobs): |
| 103 | nj=nj+self.first_jobID |
| 104 | output=[] |
| 105 | if njc == self.ncjobs : break |
| 106 | |
| 107 | msg += "... job # %s "%str(nj+1) |
| 108 | listRunField.append(run_jobToSave) |
| 109 | |
| 110 | # Prepare configuration file |
| 111 | |
| 112 | output.append('out_files_'+str(nj+1)+'.tgz') |
| 113 | job_ToSave={'outputFiles': output} |
| 114 | listField.append(job_ToSave) |
| 115 | |
| 116 | listID.append(nj+1) |
| 117 | njc = njc + 1 |
| 118 | pass |
| 119 | common.logger.log(10-1,msg) |
| 120 | # ## Not clear why here.. DS |
| 121 | # self.job_type.setArgsList() |
| 122 | common._db.updateRunJob_(listID , listRunField ) |
| 123 | common._db.updateJob_(listID, listField ) |
| 124 | |
| 125 | if self.isNew: |
| 126 | # Create script (sh) |
| 127 | script_writer.modifyTemplateScript() |
| 128 | |
| 129 | # SL This should be a list, rather than a string! |
| 130 | concString = ',' |
| 131 | inSand='' |
| 132 | if len(self.job_type.inputSandbox(1)): |
| 133 | inSand += concString.join(self.job_type.inputSandbox(1)) |
| 134 | # Sandbox, Start Dir , outputDir |
| 135 | param = {'globalSandbox': inSand , 'startDirectory': common.work_space.cwdDir() , 'outputDirectory': common.work_space.resDir() } |
| 136 | common._db.updateTask_(param) |
| 137 | # define requirement |
| 138 | common.scheduler.sched_fix_parameter() |
| 139 | common.scheduler.declare(listID) |
| 140 | |
| 141 | stop = time.time() |
| 142 | common.logger.info('Creating '+str(self.total_njobs)+' jobs, please wait...') |
| 143 | |
| 144 | stop = time.time() |
| 145 | common.logger.debug( "Creation Time: "+str(stop - start)) |
| 146 | |
| 147 | msg = 'Total of %d jobs created'%njc |
| 148 | if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs |
| 149 | msg = msg + '.\n' |
| 150 | common.logger.info(msg) |
| 151 | |
| 152 | return |