| Revision: | 1.75 |
| Committed: | Fri May 29 15:10:50 2009 UTC (15 years, 11 months ago) by spiga |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_6_6_pre6, CRAB_2_7_1_pre2, CRAB_2_6_6_pre5, CRAB_2_7_1_pre1, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_7_0, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, CRAB_2_6_0_pre7, CRAB_2_6_0_pre6, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Changes since 1.74: | +3 -2 lines |
| Log Message: | improved logging |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | nsmirnov | 1.1 | from Actor import Actor |
| 2 | from WorkSpace import WorkSpace | ||
| 3 | from JobList import JobList | ||
| 4 | from ScriptWriter import ScriptWriter | ||
| 5 | from Scheduler import Scheduler | ||
| 6 | from crab_exceptions import * | ||
| 7 | from crab_util import * | ||
| 8 | import common | ||
| 9 | |||
| 10 | slacapra | 1.8 | import os, string, math |
| 11 | slacapra | 1.45 | import time |
| 12 | |||
| 13 | nsmirnov | 1.1 | class Creator(Actor): |
| 14 | spiga | 1.70 | def __init__(self, job_type_name, cfg_params, ncjobs, skip_blocks=False,isNew=True,first_jobID=0): |
| 15 | nsmirnov | 1.1 | self.job_type_name = job_type_name |
| 16 | corvo | 1.30 | self.job_type = None |
| 17 | nsmirnov | 1.1 | self.cfg_params = cfg_params |
| 18 | nsmirnov | 1.2 | self.total_njobs = 0 |
| 19 | self.total_number_of_events = 0 | ||
| 20 | self.job_number_of_events = 0 | ||
| 21 | slacapra | 1.7 | self.first_event = 0 |
| 22 | gutsche | 1.38 | self.jobParamsList=[] |
| 23 | spiga | 1.70 | self.first_jobID = first_jobID |
| 24 | self.isNew = isNew | ||
| 25 | |||
| 26 | self.createJobTypeObject(ncjobs,skip_blocks,self.isNew) | ||
| 27 | spiga | 1.73 | common.logger.debug( __name__+": JobType "+self.job_type.name()+" created") |
| 28 | nsmirnov | 1.1 | |
| 29 | |||
| 30 | gutsche | 1.38 | self.total_njobs = self.job_type.numberOfJobs(); |
| 31 | spiga | 1.73 | common.logger.debug( __name__+": total # of jobs = "+`self.total_njobs`) |
| 32 | nsmirnov | 1.1 | |
| 33 | self.ncjobs = ncjobs | ||
| 34 | if ncjobs == 'all' : self.ncjobs = self.total_njobs | ||
| 35 | if ncjobs > self.total_njobs : self.ncjobs = self.total_njobs | ||
| 36 | spiga | 1.11 | |
| 37 | spiga | 1.60 | self.job_type_name = self.job_type.name() |
| 38 | |||
| 39 | spiga | 1.73 | common.logger.debug( "Creator constructor finished") |
| 40 | nsmirnov | 1.1 | return |
| 41 | |||
| 42 | spiga | 1.70 | def writeJobsSpecsToDB(self, first_jobID=0): |
| 43 | slacapra | 1.7 | """ |
| 44 | Write firstEvent and maxEvents in the DB for future use | ||
| 45 | """ | ||
| 46 | |||
| 47 | spiga | 1.70 | self.job_type.split(self.jobParamsList,first_jobID) |
| 48 | nsmirnov | 1.2 | return |
| 49 | |||
| 50 | nsmirnov | 1.1 | def nJobs(self): |
| 51 | return self.total_njobs | ||
| 52 | |||
| 53 | spiga | 1.61 | |
| 54 | def nJobsL(self): | ||
| 55 | jobsL=[] | ||
| 56 | for i in range(self.total_njobs): | ||
| 57 | spiga | 1.70 | jobsL.append(self.first_jobID+i+1) |
| 58 | spiga | 1.61 | return jobsL |
| 59 | |||
| 60 | spiga | 1.70 | def createJobTypeObject(self,ncjobs,skip_blocks,isNew): |
| 61 | nsmirnov | 1.1 | file_name = 'cms_'+ string.lower(self.job_type_name) |
| 62 | klass_name = string.capitalize(self.job_type_name) | ||
| 63 | |||
| 64 | try: | ||
| 65 | klass = importName(file_name, klass_name) | ||
| 66 | except KeyError: | ||
| 67 | msg = 'No `class '+klass_name+'` found in file `'+file_name+'.py`' | ||
| 68 | raise CrabException(msg) | ||
| 69 | except ImportError, e: | ||
| 70 | msg = 'Cannot create job type '+self.job_type_name | ||
| 71 | msg += ' (file: '+file_name+', class '+klass_name+'):\n' | ||
| 72 | msg += str(e) | ||
| 73 | raise CrabException(msg) | ||
| 74 | |||
| 75 | spiga | 1.70 | self.job_type = klass(self.cfg_params,ncjobs,skip_blocks,isNew) |
| 76 | nsmirnov | 1.1 | return |
| 77 | nsmirnov | 1.5 | |
| 78 | def jobType(self): | ||
| 79 | return self.job_type | ||
| 80 | nsmirnov | 1.1 | |
| 81 | def run(self): | ||
| 82 | nsmirnov | 1.4 | """ |
| 83 | The main method of the class. | ||
| 84 | """ | ||
| 85 | nsmirnov | 1.1 | |
| 86 | spiga | 1.73 | common.logger.debug( "Creator::run() called") |
| 87 | slacapra | 1.45 | start = time.time() |
| 88 | nsmirnov | 1.1 | # Instantiate ScriptWriter |
| 89 | spiga | 1.69 | script_writer = ScriptWriter(self.cfg_params,'crab_template.sh') |
| 90 | nsmirnov | 1.1 | |
| 91 | # Loop over jobs | ||
| 92 | slacapra | 1.44 | argsList = [] |
| 93 | nsmirnov | 1.1 | njc = 0 |
| 94 | spiga | 1.60 | listID=[] |
| 95 | listField=[] | ||
| 96 | spiga | 1.63 | listRunField=[] |
| 97 | spiga | 1.67 | run_jobToSave = {'status' :'C', \ |
| 98 | mcinquil | 1.72 | 'statusScheduler' : 'Created', \ |
| 99 | 'state' : "Created" } | ||
| 100 | |||
| 101 | spiga | 1.75 | msg = ("Creating jobs :\n") |
| 102 | nsmirnov | 1.4 | for nj in range(self.total_njobs): |
| 103 | spiga | 1.70 | nj=nj+self.first_jobID |
| 104 | spiga | 1.64 | output=[] |
| 105 | nsmirnov | 1.1 | if njc == self.ncjobs : break |
| 106 | |||
| 107 | spiga | 1.75 | msg += "... job # %s "%str(nj+1) |
| 108 | spiga | 1.63 | listRunField.append(run_jobToSave) |
| 109 | nsmirnov | 1.3 | |
| 110 | nsmirnov | 1.1 | # Prepare configuration file |
| 111 | |||
| 112 | spiga | 1.64 | output.append('out_files_'+str(nj+1)+'.tgz') |
| 113 | job_ToSave={'outputFiles': output} | ||
| 114 | spiga | 1.60 | listField.append(job_ToSave) |
| 115 | spiga | 1.57 | |
| 116 | spiga | 1.62 | listID.append(nj+1) |
| 117 | nsmirnov | 1.1 | njc = njc + 1 |
| 118 | pass | ||
| 119 | spiga | 1.75 | common.logger.log(10-1,msg) |
| 120 | spiga | 1.60 | # ## Not clear why here.. DS |
| 121 | # self.job_type.setArgsList() | ||
| 122 | spiga | 1.70 | common._db.updateRunJob_(listID , listRunField ) |
| 123 | common._db.updateJob_(listID, listField ) | ||
| 124 | spiga | 1.57 | |
| 125 | spiga | 1.70 | if self.isNew: |
| 126 | # Create script (sh) | ||
| 127 | script_writer.modifyTemplateScript() | ||
| 128 | |||
| 129 | # SL This should be a list, rather than a string! | ||
| 130 | concString = ',' | ||
| 131 | inSand='' | ||
| 132 | if len(self.job_type.inputSandbox(1)): | ||
| 133 | inSand += concString.join(self.job_type.inputSandbox(1)) | ||
| 134 | # Sandbox, Start Dir , outputDir | ||
| 135 | param = {'globalSandbox': inSand , 'startDirectory': common.work_space.cwdDir() , 'outputDirectory': common.work_space.resDir() } | ||
| 136 | common._db.updateTask_(param) | ||
| 137 | # define requirement | ||
| 138 | common.scheduler.sched_fix_parameter() | ||
| 139 | common.scheduler.declare(listID) | ||
| 140 | spiga | 1.60 | |
| 141 | stop = time.time() | ||
| 142 | spiga | 1.73 | common.logger.info('Creating '+str(self.total_njobs)+' jobs, please wait...') |
| 143 | fanzago | 1.12 | |
| 144 | slacapra | 1.45 | stop = time.time() |
| 145 | spiga | 1.73 | common.logger.debug( "Creation Time: "+str(stop - start)) |
| 146 | slacapra | 1.45 | |
| 147 | spiga | 1.74 | msg = 'Total of %d jobs created'%njc |
| 148 | nsmirnov | 1.1 | if njc != self.ncjobs: msg = msg + ' from %d requested'%self.ncjobs |
| 149 | msg = msg + '.\n' | ||
| 150 | spiga | 1.73 | common.logger.info(msg) |
| 151 | slacapra | 1.45 | |
| 152 | nsmirnov | 1.1 | return |