| Revision: | 1.8 |
| Committed: | Tue Jun 9 13:12:06 2009 UTC (15 years, 10 months ago) by slacapra |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_8_patch2, CRAB_2_7_9_pre1, CRAB_2_7_8_patch2_pre1, CRAB_2_7_8_patch1, CRAB_2_7_8_patch1_pre1, CRAB_2_7_8, CRAB_2_7_8_pre3, CRAB_2_7_8_pre2, CRAB_2_7_8_dash3, CRAB_2_7_8_dash2, CRAB_2_7_8_dash, CRAB_2_7_7_patch1, CRAB_2_7_7_patch1_pre1, CRAB_2_7_8_pre1, CRAB_2_7_7, CRAB_2_7_7_pre2, CRAB_2_7_7_pre1, CRAB_2_7_6_patch1, CRAB_2_7_6, CRAB_2_7_6_pre1, CRAB_2_7_5_patch1, CRAB_2_7_5, CRAB_2_7_5_pre3, CRAB_2_7_5_pre2, CRAB_2_7_5_pre1, CRAB_2_7_4_patch1, CRAB_2_7_4, CRAB_2_7_4_pre6, CRAB_2_7_4_pre5, CRAB_2_7_4_pre4, CRAB_2_7_4_pre3, CRAB_2_7_4_pre2, CRAB_2_7_4_pre1, CRAB_2_7_3, CRAB_2_7_3_pre3, CRAB_2_7_3_pre3_beta, CRAB_2_7_3_pre2, CRAB_2_7_3_pre2_beta, CRAB_2_7_3_pre1, CRAB_2_7_3_beta3, CRAB_2_7_3_beta2, CRAB_2_7_3_beta1, CRAB_2_7_3_beta, CRAB_2_7_2_p1, CRAB_2_7_1_branch_firstMERGE, CRAB_2_7_2, CRAB_2_7_2_pre4, CRAB_2_7_2_pre3, CRAB_2_7_2_pre2, CRAB_2_7_2_pre1, CRAB_2_7_1, fede_170310, CRAB_2_7_1_pre12, CRAB_2_7_1_pre11, CRAB_2_7_1_pre10, CRAB_2_7_1_pre9, CRAB_LumiMask, CRAB_2_7_lumi, from_LimiMask, CRAB_2_7_1_pre8, CRAB_2_7_1_pre6, CRAB_2_7_1_pre5, CRAB_2_7_1_wmbs_pre4, CRAB_2_7_1_pre4, CRAB_2_7_1_pre3, CRAB_2_6_6_pre6, CRAB_2_7_1_pre2, CRAB_2_6_6_pre5, CRAB_2_7_1_pre1, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_7_0, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8, HEAD |
| Branch point for: | CRAB_multiout, CRAB_2_7_1_branch, Lumi2_8, CRAB_2_6_X_br |
| Changes since 1.7: | +1 -1 lines |
| Log Message: | Introduce a CrabLogger class which hanlde the std python logging |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | spiga | 1.1 | from Actor import * |
| 2 | from crab_util import * | ||
| 3 | import common | ||
| 4 | |||
| 5 | import os, errno, time, sys, re | ||
| 6 | import commands | ||
| 7 | |||
| 8 | class JdlWriter( Actor ): | ||
| 9 | def __init__(self, cfg_params, jobs): | ||
| 10 | self.cfg_params = cfg_params | ||
| 11 | self.nj_list = jobs | ||
| 12 | slacapra | 1.3 | from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser |
| 13 | spiga | 1.6 | seWhiteList = cfg_params.get('GRID.se_white_list',[]) |
| 14 | seBlackList = cfg_params.get('GRID.se_black_list',[]) | ||
| 15 | slacapra | 1.8 | self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger()) |
| 16 | slacapra | 1.4 | self.datasetpath=self.cfg_params['CMSSW.datasetpath'] |
| 17 | if string.lower(self.datasetpath)=='none': | ||
| 18 | self.datasetpath = None | ||
| 19 | slacapra | 1.3 | |
| 20 | slacapra | 1.2 | return |
| 21 | spiga | 1.1 | |
| 22 | def run(self): | ||
| 23 | slacapra | 1.2 | """ |
| 24 | The main method of the class: write JDL for jobs in range self.nj_list | ||
| 25 | """ | ||
| 26 | spiga | 1.5 | common.logger.debug( "JdlWriter::run() called") |
| 27 | spiga | 1.1 | |
| 28 | start = time.time() | ||
| 29 | |||
| 30 | jobs_L = self.listOfjobs() | ||
| 31 | |||
| 32 | self.writer(jobs_L) | ||
| 33 | |||
| 34 | stop = time.time() | ||
| 35 | |||
| 36 | spiga | 1.5 | common.logger.log(10-1,"JDL writing time :"+str(stop - start)) |
| 37 | spiga | 1.1 | |
| 38 | slacapra | 1.2 | return |
| 39 | spiga | 1.1 | |
| 40 | |||
| 41 | def listOfjobs(self): | ||
| 42 | |||
| 43 | ### define here the list of distinct destinations sites list | ||
| 44 | distinct_dests = common._db.queryDistJob_Attr('dlsDestination', 'jobId' ,self.nj_list) | ||
| 45 | |||
| 46 | |||
| 47 | ### define here the list of jobs Id for each distinct list of sites | ||
| 48 | self.sub_jobs =[] # list of jobs Id list to submit | ||
| 49 | jobs_to_match =[] # list of jobs Id to match | ||
| 50 | all_jobs=[] | ||
| 51 | count=0 | ||
| 52 | for distDest in distinct_dests: | ||
| 53 | slacapra | 1.3 | dest = self.blackWhiteListParser.cleanForBlackWhiteList(distDest) |
| 54 | slacapra | 1.4 | if not dest and self.datasetpath: |
| 55 | fanzago | 1.7 | common.logger.info('No destination available: will not create jdl \n' ) |
| 56 | slacapra | 1.3 | continue |
| 57 | slacapra | 1.2 | all_jobs.append(common._db.queryAttrJob({'dlsDestination':distDest},'jobId')) |
| 58 | sub_jobs_temp=[] | ||
| 59 | for i in self.nj_list: | ||
| 60 | if i in all_jobs[count]: sub_jobs_temp.append(i) | ||
| 61 | if len(sub_jobs_temp)>0: | ||
| 62 | self.sub_jobs.append(sub_jobs_temp) | ||
| 63 | count +=1 | ||
| 64 | spiga | 1.1 | return self.sub_jobs |
| 65 | |||
| 66 | def writer(self,list): | ||
| 67 | """ | ||
| 68 | Materialize JDL into file | ||
| 69 | """ | ||
| 70 | slacapra | 1.3 | if len(list)==0: |
| 71 | spiga | 1.5 | common.logger.info('No destination available for any job: will not create jdl \n' ) |
| 72 | slacapra | 1.3 | |
| 73 | spiga | 1.1 | task = common._db.getTask() |
| 74 | c1 = 1 | ||
| 75 | c2 = 1 | ||
| 76 | for sub_list in list: | ||
| 77 | slacapra | 1.2 | jdl = common.scheduler.writeJDL(sub_list, task) |
| 78 | spiga | 1.1 | |
| 79 | slacapra | 1.2 | for stri in jdl: |
| 80 | #self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl' | ||
| 81 | self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl' | ||
| 82 | j_file = open(common.work_space.shareDir()+'/'+self.jdlFile, 'w') | ||
| 83 | j_file.write( stri ) | ||
| 84 | j_file.close() | ||
| 85 | c2 += 1 | ||
| 86 | c1 += 1 | ||
| 87 | spiga | 1.1 | |
| 88 | spiga | 1.5 | common.logger.info('JDL files are written to '+str(common.work_space.shareDir())+'File-*.jdl \n' ) |
| 89 | spiga | 1.1 | |
| 90 | return |