| 1 |
spiga |
1.1 |
from Actor import *
|
| 2 |
|
|
from crab_util import *
|
| 3 |
|
|
import common
|
| 4 |
|
|
|
| 5 |
|
|
import os, errno, time, sys, re
|
| 6 |
|
|
import commands
|
| 7 |
|
|
|
| 8 |
|
|
class JdlWriter( Actor ):
|
| 9 |
|
|
def __init__(self, cfg_params, jobs):
|
| 10 |
|
|
self.cfg_params = cfg_params
|
| 11 |
|
|
self.nj_list = jobs
|
| 12 |
slacapra |
1.3 |
from WMCore.SiteScreening.BlackWhiteListParser import SEBlackWhiteListParser
|
| 13 |
spiga |
1.6 |
seWhiteList = cfg_params.get('GRID.se_white_list',[])
|
| 14 |
|
|
seBlackList = cfg_params.get('GRID.se_black_list',[])
|
| 15 |
slacapra |
1.3 |
self.blackWhiteListParser = SEBlackWhiteListParser(seWhiteList, seBlackList, common.logger)
|
| 16 |
slacapra |
1.4 |
self.datasetpath=self.cfg_params['CMSSW.datasetpath']
|
| 17 |
|
|
if string.lower(self.datasetpath)=='none':
|
| 18 |
|
|
self.datasetpath = None
|
| 19 |
slacapra |
1.3 |
|
| 20 |
slacapra |
1.2 |
return
|
| 21 |
spiga |
1.1 |
|
| 22 |
|
|
def run(self):
|
| 23 |
slacapra |
1.2 |
"""
|
| 24 |
|
|
The main method of the class: write JDL for jobs in range self.nj_list
|
| 25 |
|
|
"""
|
| 26 |
spiga |
1.5 |
common.logger.debug( "JdlWriter::run() called")
|
| 27 |
spiga |
1.1 |
|
| 28 |
|
|
start = time.time()
|
| 29 |
|
|
|
| 30 |
|
|
jobs_L = self.listOfjobs()
|
| 31 |
|
|
|
| 32 |
|
|
self.writer(jobs_L)
|
| 33 |
|
|
|
| 34 |
|
|
stop = time.time()
|
| 35 |
|
|
|
| 36 |
spiga |
1.5 |
common.logger.log(10-1,"JDL writing time :"+str(stop - start))
|
| 37 |
spiga |
1.1 |
|
| 38 |
slacapra |
1.2 |
return
|
| 39 |
spiga |
1.1 |
|
| 40 |
|
|
|
| 41 |
|
|
def listOfjobs(self):
|
| 42 |
|
|
|
| 43 |
|
|
### define here the list of distinct destinations sites list
|
| 44 |
|
|
distinct_dests = common._db.queryDistJob_Attr('dlsDestination', 'jobId' ,self.nj_list)
|
| 45 |
|
|
|
| 46 |
|
|
|
| 47 |
|
|
### define here the list of jobs Id for each distinct list of sites
|
| 48 |
|
|
self.sub_jobs =[] # list of jobs Id list to submit
|
| 49 |
|
|
jobs_to_match =[] # list of jobs Id to match
|
| 50 |
|
|
all_jobs=[]
|
| 51 |
|
|
count=0
|
| 52 |
|
|
for distDest in distinct_dests:
|
| 53 |
slacapra |
1.3 |
dest = self.blackWhiteListParser.cleanForBlackWhiteList(distDest)
|
| 54 |
slacapra |
1.4 |
if not dest and self.datasetpath:
|
| 55 |
spiga |
1.5 |
common.logger.info'No destination available: will not create jdl \n' )
|
| 56 |
slacapra |
1.3 |
continue
|
| 57 |
slacapra |
1.2 |
all_jobs.append(common._db.queryAttrJob({'dlsDestination':distDest},'jobId'))
|
| 58 |
|
|
sub_jobs_temp=[]
|
| 59 |
|
|
for i in self.nj_list:
|
| 60 |
|
|
if i in all_jobs[count]: sub_jobs_temp.append(i)
|
| 61 |
|
|
if len(sub_jobs_temp)>0:
|
| 62 |
|
|
self.sub_jobs.append(sub_jobs_temp)
|
| 63 |
|
|
count +=1
|
| 64 |
spiga |
1.1 |
return self.sub_jobs
|
| 65 |
|
|
|
| 66 |
|
|
def writer(self,list):
|
| 67 |
|
|
"""
|
| 68 |
|
|
Materialize JDL into file
|
| 69 |
|
|
"""
|
| 70 |
slacapra |
1.3 |
if len(list)==0:
|
| 71 |
spiga |
1.5 |
common.logger.info('No destination available for any job: will not create jdl \n' )
|
| 72 |
slacapra |
1.3 |
|
| 73 |
spiga |
1.1 |
task = common._db.getTask()
|
| 74 |
|
|
c1 = 1
|
| 75 |
|
|
c2 = 1
|
| 76 |
|
|
for sub_list in list:
|
| 77 |
slacapra |
1.2 |
jdl = common.scheduler.writeJDL(sub_list, task)
|
| 78 |
spiga |
1.1 |
|
| 79 |
slacapra |
1.2 |
for stri in jdl:
|
| 80 |
|
|
#self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl'
|
| 81 |
|
|
self.jdlFile='File-'+str(c1)+'_'+str(c2)+'.jdl'
|
| 82 |
|
|
j_file = open(common.work_space.shareDir()+'/'+self.jdlFile, 'w')
|
| 83 |
|
|
j_file.write( stri )
|
| 84 |
|
|
j_file.close()
|
| 85 |
|
|
c2 += 1
|
| 86 |
|
|
c1 += 1
|
| 87 |
spiga |
1.1 |
|
| 88 |
spiga |
1.5 |
common.logger.info('JDL files are written to '+str(common.work_space.shareDir())+'File-*.jdl \n' )
|
| 89 |
spiga |
1.1 |
|
| 90 |
|
|
return
|
