| Revision: | 1.8 |
| Committed: | Tue Jun 9 13:12:06 2009 UTC (15 years, 10 months ago) by slacapra |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_6_6_pre6, CRAB_2_7_1_pre2, CRAB_2_6_6_pre5, CRAB_2_7_1_pre1, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_7_0, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Changes since 1.7: | +1 -1 lines |
| Log Message: | Introduce a CrabLogger class which hanlde the std python logging |
| # | Content |
|---|---|
| 1 | from Actor import * |
| 2 | from crab_util import * |
| 3 | import common |
| 4 | |
| 5 | import os, errno, time, sys, re |
| 6 | import commands |
| 7 | |
| 8 | class JdlWriter( Actor ): |
| 9 | def __init__(self, cfg_params, jobs): |
| 10 | self.cfg_params = cfg_params |
| 11 | self.nj_list = jobs |
| 12 | from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser |
| 13 | seWhiteList = cfg_params.get('GRID.se_white_list',[]) |
| 14 | seBlackList = cfg_params.get('GRID.se_black_list',[]) |
| 15 | self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger()) |
| 16 | self.datasetpath=self.cfg_params['CMSSW.datasetpath'] |
| 17 | if string.lower(self.datasetpath)=='none': |
| 18 | self.datasetpath = None |
| 19 | |
| 20 | return |
| 21 | |
| 22 | def run(self): |
| 23 | """ |
| 24 | The main method of the class: write JDL for jobs in range self.nj_list |
| 25 | """ |
| 26 | common.logger.debug( "JdlWriter::run() called") |
| 27 | |
| 28 | start = time.time() |
| 29 | |
| 30 | jobs_L = self.listOfjobs() |
| 31 | |
| 32 | self.writer(jobs_L) |
| 33 | |
| 34 | stop = time.time() |
| 35 | |
| 36 | common.logger.log(10-1,"JDL writing time :"+str(stop - start)) |
| 37 | |
| 38 | return |
| 39 | |
| 40 | |
| 41 | def listOfjobs(self): |
| 42 | |
| 43 | ### define here the list of distinct destinations sites list |
| 44 | distinct_dests = common._db.queryDistJob_Attr('dlsDestination', 'jobId' ,self.nj_list) |
| 45 | |
| 46 | |
| 47 | ### define here the list of jobs Id for each distinct list of sites |
| 48 | self.sub_jobs =[] # list of jobs Id list to submit |
| 49 | jobs_to_match =[] # list of jobs Id to match |
| 50 | all_jobs=[] |
| 51 | count=0 |
| 52 | for distDest in distinct_dests: |
| 53 | dest = self.blackWhiteListParser.cleanForBlackWhiteList(distDest) |
| 54 | if not dest and self.datasetpath: |
| 55 | common.logger.info('No destination available: will not create jdl \n' ) |
| 56 | continue |
| 57 | all_jobs.append(common._db.queryAttrJob({'dlsDestination':distDest},'jobId')) |
| 58 | sub_jobs_temp=[] |
| 59 | for i in self.nj_list: |
| 60 | if i in all_jobs[count]: sub_jobs_temp.append(i) |
| 61 | if len(sub_jobs_temp)>0: |
| 62 | self.sub_jobs.append(sub_jobs_temp) |
| 63 | count +=1 |
| 64 | return self.sub_jobs |
| 65 | |
| 66 | def writer(self,list): |
| 67 | """ |
| 68 | Materialize JDL into file |
| 69 | """ |
| 70 | if len(list)==0: |
| 71 | common.logger.info('No destination available for any job: will not create jdl \n' ) |
| 72 | |
| 73 | task = common._db.getTask() |
| 74 | c1 = 1 |
| 75 | c2 = 1 |
| 76 | for sub_list in list: |
| 77 | jdl = common.scheduler.writeJDL(sub_list, task) |
| 78 | |
| 79 | for stri in jdl: |
| 80 | #self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl' |
| 81 | self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl' |
| 82 | j_file = open(common.work_space.shareDir()+'/'+self.jdlFile, 'w') |
| 83 | j_file.write( stri ) |
| 84 | j_file.close() |
| 85 | c2 += 1 |
| 86 | c1 += 1 |
| 87 | |
| 88 | common.logger.info('JDL files are written to '+str(common.work_space.shareDir())+'File-*.jdl \n' ) |
| 89 | |
| 90 | return |