Processing/bin/submit.py

#!/usr/bin/env python
#---------------------------------------------------------------------------------------------------
# Script to test, create and submit one complete production task
#
# Complete refers here to the proper preparation of the ultimate storage location (a storge element
# with a given storage path etc.), the submission of a test job to evaluate timing and data output
# size and finally the creation of the job configurations and the submission of the jobs to the grid
# sites. The grid is accessed via crab tools.
#
# While the script is pretty complete it has become a bit long and messy. Some cleanup will
# hopefully soon be performed.
#
# Author: C.Paus                                                                      (July 1, 2008)
#---------------------------------------------------------------------------------------------------
import os,sys,getopt,re,string
import task,translator

#===================================================================================================
def domain():
    d = os.uname()[1]
    f = d.split('.')
    return '.'.join(f[1:])

def storage(seFile):
    # decide on the forseen default storage place (where are we running?)
    storageTag = 'T2_US_MIT'
    dom = domain()
    if   re.search('mit.edu',dom):
        storageTag = 'T2_US_MIT'
    elif re.search('cern.ch',dom):
        storageTag = 'T0_CH_CERN'
    cmd = 'grep ^' + storageTag + ' ' + seFile
    for line in os.popen(cmd).readlines():   # run command
        #print ' LINE: ' + line
        line = line[:-1]                     # strip '\n'
        line = line.replace(' ','')
        f = line.split(':')
        storageEle    = f[1] 
        storagePath   = f[2] 
        userRemoteDir = f[3] 
    return storageEle + ':' + storagePath + ':' + userRemoteDir

#===================================================================================================
def getFiles(mitCfg,version):
    # Prepare file based processing input
    runFile  =  os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/' + version + '/' + 'run.sh'
    if not os.path.exists(runFile):
        cmd = "Run file not found: %s" % runFile
        raise RuntimeError, cmd
    cmd = 'cp ' + runFile + ' ./'
    os.system(cmd)
    writeCfgFile = os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/' + version + '/' + 'writeCfg.py'
    cmd = 'cp ' + writeCfgFile + ' ./'
    if not os.path.exists(writeCfgFile):
        cmd = "Write Cfg file not found: %s" % writeCfgFile
        raise RuntimeError, cmd
    os.system(cmd)
    storageFile  =  os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/' + version + '/' \
                   + 'storageTarget'
    if not os.path.exists(storageFile):
        cmd = "Storage file not found: %s" % storageFile
        raise RuntimeError, cmd
    cmd = 'cp ' + storageFile + ' ./'
    os.system(cmd)

#===================================================================================================
def makeLfnFile(mitCfg,version,mitDataset,dbs,useExistingLfns):
    lfnFile  = mitCfg + '/' + version + '/' + mitDataset + '.lfns'
    if os.path.exists(lfnFile):
        print "\n INFO -- Lfn file found: %s. Someone already worked on this dataset.\n" % lfnFile
        if not useExistingLfns:
            cmd = 'rm ' + lfnFile
            os.system(cmd)
    
    # recreate if requested or not existing
    if not useExistingLfns or not os.path.exists(lfnFile):
        cmd = 'input.py --dbs=' + dbs + ' --option=lfn --dataset=' + cmsDataset \
              + ' | sort -u > ' + lfnFile
        print ' Input: ' + cmd + '\n'
        os.system(cmd)

    return lfnFile

#===================================================================================================
def makeSiteFile(mitCfg,version,mitDataset,dbs,useExistingSites):
    siteFile  = mitCfg + '/' + version + '/' + mitDataset + '.sites'
    if os.path.exists(siteFile):
        print "\n INFO -- Site file found: %s. Someone already worked on this dataset.\n" % siteFile
        if not useExistingSites:
            cmd = 'rm ' + siteFile
            os.system(cmd)
    
    # recreate if requested or not existing
    if not useExistingSites or not os.path.exists(siteFile):
        cmd = 'sitesList.py --dbs=' + dbs + ' --dataset=' + cmsDataset + ' > ' + siteFile
        print ' Sites: ' + cmd + '\n'
        os.system(cmd)

    return siteFile

#===================================================================================================
def searchReplace(line,mitCfg,version,mitDataset,storage, \
                  cmsDataset='X',cmsswPy='X',dbs='X',sched='X',blacklist='X',skpEvts='X'):
    # decode storage variable
    st = storage.split(':')
    # compile search and replacement sequences
    pCmsDset   = re.compile('XX-CMSDATASET-XX')
    pMitDset   = re.compile('XX-MITDATASET-XX')
    pCmsswPy   = re.compile('XX-CMSSWPY-XX')
    pMitCfg    = re.compile('XX-MITCFG-XX')
    pMitVers   = re.compile('XX-MITVERSION-XX')
    pMitSe     = re.compile('XX-MITSE-XX')
    pMitSPath  = re.compile('XX-MITSPATH-XX')
    pMitRDir   = re.compile('XX-MITRDIR-XX')
    pDbs       = re.compile('XX-DBS-XX')
    pSched     = re.compile('XX-SCHED-XX')
    pBlacklist = re.compile('XX-BLACKLIST-XX')
    pSkpEvts   = re.compile('XX-SKIPEVTS-XX')
    # perform all search and replaces
    line = pCmsDset  .sub(cmsDataset,  line);
    line = pMitDset  .sub(mitDataset,  line);
    line = pCmsswPy  .sub(cmsswPy,     line);
    line = pMitCfg   .sub(mitCfg,      line);
    line = pMitVers  .sub(version,     line);
    line = pMitSe    .sub(st[0],       line);
    line = pMitSPath .sub(st[1],       line);
    line = pMitRDir  .sub(st[2],       line);
    line = pDbs      .sub(dbs,         line);
    line = pSched    .sub(sched,       line);
    line = pBlacklist.sub(blacklist,   line);
    return line

#===================================================================================================
def adjustCfg(line,nevents,crabId):
    # compile search and replacement sequences
    pNevents   = re.compile('XX-NEVENTS-XX')
    pCrabId    = re.compile('XX-CRABID-XX')
    # perform all search and replaces
    line = pNevents  .sub(str(nevents),line);
    line = pCrabId   .sub(crabId,      line);
    return line

#===================================================================================================
def findStoragePath(mitCfg,version,mitDataset,seFile):
    # find the forseen storage place
    cmd = 'grep ^storage_path ' + os.environ['MIT_PROD_DIR'] \
          + '/' + mitCfg + '/' + version + '/crab.cfg'
    for file in os.popen(cmd).readlines():
        line         = file[:-1]
        line         = searchReplace(line,mitCfg,version,mitDataset,storage(seFile));
        # decode the storage directory name
        names        = line.split("=")
        names        = names[1:]
        storagePath  = "=".join(names)
        storagePath  = re.sub("\s", "",storagePath)
    cmd = 'grep ^user_remote_dir ' + os.environ['MIT_PROD_DIR'] \
          + '/' + mitCfg + '/' + version + '/crab.cfg'
    for file in os.popen(cmd).readlines():
        line         = file[:-1]
        line         = searchReplace(line,mitCfg,version,mitDataset,storage(seFile));
        # decode the storage directory name
        names        = line.split(" ")
        storagePath += names[-1]
    return storagePath

#===================================================================================================
def createDirCern(storagePath):
    # check whether path exists
    cmd = 'rfdir ' + storagePath + ' >& /dev/null'
    status = os.system(cmd)
    
    # create it if missing
    if status == 0:
        print ' Castor directory exists: ' + storagePath + '\n  --> Moving on.'
    else:
        print ' Castor directory needs to be created.'
        cmd = 'rfmkdir -p ' + storagePath
        status = os.system(cmd)
        if status == 0:
            print '  --> Created: ' + storagePath
            
        # always set the permissions
    cmd = 'rfchmod 777 ' + storagePath
    status = os.system(cmd)
    if status == 0:
        print '  --> Set permissions: 777\n'
    else:
        print '  --> Setting permissions failed. EXIT now.\n'
        sys.exit(1)

#===================================================================================================
def create(path):
    status = -1
    if re.search('/pnfs/cmsaf.mit.edu/t2bat',path):
        f    = path.split('=')
        path = f[-1]
        cmd = 'ssh -x paus@cgate mkdir -p  ' + path
        status = os.system(cmd)
        cmd = 'ssh -x paus@cgate chmod 777 ' + path
        status = os.system(cmd)
    return status

#===================================================================================================
def createDirGeneral(storageEle,storagePath,fastCreate):
    # create all relevant subdirectories
    f = storagePath.split('/')                    # splitting every '/'
    storagePath2 = "/".join(f[:-1])
    storagePath1 = "/".join(f[:-2])
    storagePath0 = "/".join(f[:-3])

    if create(storagePath) == 0:
        print ' '
        print ' Directory was created at MIT.\n'
        return

    # set the storage URL
    storageUrl = 'srm://' + storageEle + ':8443' + storagePath

    # check whether path exists
    cmd = 'srmls ' + storageUrl + ' >& /dev/null'
    status = os.system(cmd)
    
    # create it only if missing
    if status == 0:
        print ' '
        print ' Directory already found.... moving on.'

    else:
        # create all relevant directories
        if fastCreate == 0:
            cmd = 'srmmkdir srm://' + storageEle + ':8443' + storagePath0 + ' >& /dev/null'
            print '  srmmkdir: ' + cmd
            status = os.system(cmd)
            print '  srmmkdir: status %d'%(status)
            cmd = 'srmmkdir srm://' + storageEle + ':8443' + storagePath1 + ' >& /dev/null'
            print '  srmmkdir: ' + cmd
            status = os.system(cmd)
            print '  srmmkdir: status %d'%(status)
            cmd = 'srmmkdir srm://' + storageEle + ':8443' + storagePath2 + ' >& /dev/null'
            print '  srmmkdir: ' + cmd
            status = os.system(cmd)
            print '  srmmkdir: status %d'%(status)
        else:
            print ' Fast create activated. '
    
        # create the main storage directory
        cmd = 'srmmkdir srm://' + storageEle + ':8443' + storagePath  + ' >& /dev/null'
        print '  srmmkdir: ' + cmd
        status = os.system(cmd)
        print '  srmmkdir: status  %d'%(status)
        if status == 0:
            print ' '
            print ' Directory was created.'
        else:
            print ' '
            print ' '
            print ' '
            print ' ERROR - failed to create the Storage Area.'
            print ' '
            print ' '

    print ' Check permissions with:  srmls -l -count=1 ' + storageUrl + '\n'
    cmd = 'srmls -l -count=1 ' + storageUrl + ' | grep UserPermission | grep RWX'
    status = -1
    for line in os.popen(cmd).readlines():  # run command
        line = line[:-1]
        print " Permissions? " + line
        status = 0
       #status = os.system(cmd)
    print ' '

    if status != 0:
        print '  --> Directory creation or permissions failed. EXIT now.\n'
        sys.exit(1)
 
#===================================================================================================
# Main starts here
#===================================================================================================
# Define string to explain usage of the script
usage =  "Usage: submit.py --cmsDataset=<name> | --mitDataset=<name>\n"
usage += "                 --cmssw=<name>\n"
usage += "                 --mitCfg=<name>\n"
usage += "                 --version=<version>\n"
usage += "                 --dbs=<name>\n"
usage += "                 --sched=<name>\n"
usage += "                 --blacklist=<name>\n"
usage += "                 --nSubmit=<submittedJobs>\n"
usage += "                 --skipEvents=<'nRunX:nEventY','nRunXX:nEventYY',...>\n"
usage += "                 --fixSites=<siteList>\n"
usage += "                 --complete\n"
usage += "                 --testJob\n"
usage += "                 --noTestJob\n"
usage += "                 --help\n"

# Define the valid options which can be specified and check out the command line
valid = ['cmsDataset=','mitDataset=','cmssw=','mitCfg=','version=','dbs=','sched=','blacklist=',
         'nSubmit=','skipEvents=','fixSites=','complete','useExistingLfns','useExistingSites',
         'testJob','noTestJob','test','help']
try:
    opts, args = getopt.getopt(sys.argv[1:], "", valid)
except getopt.GetoptError, ex:
    print usage
    print str(ex)
    sys.exit(1)

# --------------------------------------------------------------------------------------------------
# Get all parameters for the production
# --------------------------------------------------------------------------------------------------
# crab id
cmd = "date +crab_0_%y%m%d_%H%M%S"
for line in os.popen(cmd).readlines():  # run command
    line = line[:-1]
    crabId = line
print "\n This job will be CrabId: " + crabId + "\n"
# other non command line parameters
fastCreate       = 0
# Set defaults for each command line parameter/option
cmsDataset       = None
mitDataset       = None
cmssw            = "cmssw"
mitCfg           = "filefi"
version          = os.environ['MIT_VERS']
#dbs              = "https://cmsdbsprod.cern.ch:8443/cms_dbs_prod_global/servlet/DBSServlet"
dbs              = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
sched            = "glite"
blacklist        = ""
nSubmit          = -1
skpEvts          = ''
fixSites         = ''
complete         = 0
useExistingLfns  = False
useExistingSites = False
noTestJob        = 0
testJob          = 0
test             = 0

# Read new values from the command line
for opt, arg in opts:
    if opt == "--help":
        print usage
        sys.exit(0)
    if opt == "--cmsDataset":
        cmsDataset       = arg
    if opt == "--mitDataset":
        mitDataset       = arg
    if opt == "--cmssw":
        cmssw            = arg
    if opt == "--mitCfg":
        mitCfg           = arg
    if opt == "--version":
        version          = arg
    if opt == "--dbs":
        dbs              = arg
    if opt == "--sched":
        sched            = arg
    if opt == "--blacklist":
        blacklist        = arg
    if opt == "--nSubmit":
        nSubmit          = arg
    if opt == "--skipEvents":
        skpEvts          = arg
    if opt == "--fixSites":
        fixSites         = arg
    if opt == "--complete":
        complete         = 1
    if opt == "--useExistingLfns":
        useExistingLfns  = True
    if opt == "--useExistingSites":
        useExistingSites = True
    if opt == "--noTestJob":
        noTestJob        = 1
        testJob          = 0
    if opt == "--testJob":
        testJob          = 1
        noTestJob        = 0
    if opt == "--test":
        test             = 1

# Deal with obvious problems
if cmsDataset == None and mitDataset == None:
    cmd = "--cmsDataset | --mitDataset  options not provided. One of them is required."
    raise RuntimeError, cmd
seFile = os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/'+ version + '/seTable'
if not os.path.exists(seFile):
    cmd = "Storage element table not found: %s" % seFile
    raise RuntimeError, cmd
crabFile = os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/' + version + '/' + 'crab.cfg'
if not os.path.exists(crabFile):
    cmd = "Crab file not found: %s" % crabFile
    raise RuntimeError, cmd
cmsswFile = os.environ['MIT_PROD_DIR'] + '/' + mitCfg + '/' + version + '/' + cmssw + '.py'
if not os.path.exists(cmsswFile):
    cmd = "Cmssw file not found: %s" % cmsswFile
    cmd = " XXXX ERROR no valid configuration found XXXX"
    raise RuntimeError, cmd
cmsswPy  = cmssw + '_' + crabId + '.py'

# Prepare the ce/se translator
trans = translator.Translator(os.environ['MIT_PROD_DIR']+'/'+mitCfg+'/'+version+'/ceTable',
                              os.environ['MIT_PROD_DIR']+'/'+mitCfg+'/'+version+'/seTable',
                              os.environ['MIT_PROD_DIR']+'/'+mitCfg+'/'+version+'/preferredSites')

# Create the corresponding crab task
crabTask             = task.Task(crabId,cmsDataset,mitDataset,mitCfg,version,cmssw)
if crabTask.mitDataset == 'undefined' or crabTask.cmsDataset == 'undefined':
    print "ERROR - dataset not defined."
    sys.exit(1)
else:
    mitDataset = crabTask.mitDataset
    cmsDataset = crabTask.cmsDataset
    dbs        = crabTask.dbs

# Deal with obvious problems
if cmsDataset == None or mitDataset == None:
    cmd = "--cmsDataset & --mitDataset  " + \
          "Have to be defined now (do you have the right database (Productions.<cmssw>)?"
    raise RuntimeError, cmd

getFiles(mitCfg,version)
lfnFile  = makeLfnFile (mitCfg,version,mitDataset,dbs,useExistingLfns)

crabTask.storagePath = findStoragePath(mitCfg,version,mitDataset,seFile)
crabTask.loadAllLfns(lfnFile)
crabTask.loadCompletedLfns()
crabTask.createMissingLfns(lfnFile,lfnFile + '_' + crabTask.tag)
# decide whether there are more jobs (lfns=files) to be submitted
if crabTask.nLfnMissing == 0:
    print ' All requested LFNs are available. EXIT now.'
    sys.exit()
# decide whether we always need to create the full directory tree
if crabTask.nLfnDone != 0:
    fastCreate = 1

# ok, looks like we will be submitting
siteFile = makeSiteFile(mitCfg,version,mitDataset,dbs,useExistingSites)

crabTask.createSubTasks(lfnFile + '_' + crabTask.tag)
cmd = 'cp ' + lfnFile +  '_' + crabTask.tag + '_*' + ' ./'
os.system(cmd)

nevents = len(crabTask.lfns)

# Say what we do now
print ' Preparing dataset: ' + cmsDataset + ' [MIT: ' + mitDataset + ' with ' + str(nevents) + \
      ' total jobs(=lfns), nEvtsTotal: %d]' % crabTask.nTotalEvts

# --------------------------------------------------------------------------------------------------
# Prepare the config files
# --------------------------------------------------------------------------------------------------
# Cleaning up
cmd = "rm -f crab_" + crabTask.tag + ".cfg crab_" + crabTask.tag + ".cfg-Template " + cmsswPy
os.system(cmd)

# Storage

# Parse template input and write the crab configuration file
fileInput  = open(crabFile,'r')
fileOutput = open("crab_" + crabTask.tag + ".cfg-Template",'w')
line = fileInput.readline()
while (line != ''):
    if line[0] != '#':
        line = searchReplace(line,mitCfg,version,mitDataset,storage(seFile), \
                             cmsDataset,cmsswPy,dbs,sched,blacklist,skpEvts)
    fileOutput.write(line)
    line = fileInput.readline()
fileInput .close()
fileOutput.close()

# Parse template input and write the crab configuration file
fileInput  = open(cmsswFile,'r')
fileOutput = open(cmsswPy,'w')
line = fileInput.readline()
while (line != ''):
    if line[0] != '#':
        line = searchReplace(line,mitCfg,version,mitDataset,storage(seFile), \
                             cmsDataset,cmsswPy,dbs,sched,blacklist,skpEvts)
    fileOutput.write(line)
    line = fileInput.readline()
fileInput .close()
fileOutput.close()

# --------------------------------------------------------------------------------------------------
# Job creation and submission
# --------------------------------------------------------------------------------------------------
print '\n Using CMSSW version: ' + os.environ['CMSSW_VERSION']
print   ' Using CRAB  version: ' + os.environ['CRAB_VERS'] + '\n'

pattern1    = 'working directory'
pattern2    = 'Total of '

# Go through the crabTask and submit each subtask to the grid
for subTask in crabTask.subTasks:
    
    tag = crabTask.tag + '_' + subTask.tag()
    print ' Working on subtask: ' + tag

    nJobsTotal  = 0
    crabIdCheck = ''

    # adjust crab config
    fileInput  = open("crab_" + crabTask.tag + ".cfg-Template",'r')
    fileOutput = open("crab_" + crabTask.tag + ".cfg",'w')
    line = fileInput.readline()
    while (line != ''):
        if line[0] != '#':
            line = adjustCfg(line,subTask.nSubTaskLfn,tag)
        fileOutput.write(line)
        line = fileInput.readline()
    fileInput .close()
    fileOutput.close()

    # ----------------------------------------------------------------------------------------------
    # Deal with storage element area
    # ----------------------------------------------------------------------------------------------
    # find the forseen storage place
    cmd = 'grep ^storage_element crab_' + crabTask.tag + '.cfg-Template'
    for file in os.popen(cmd).readlines():   # run command
        line        = file[:-1]              # strip '\n'
        # decode the storage element name
        names       = line.split("=")        # splitting every '='
        storageEle  = names.pop()
        storageEle  = re.sub("\s", "",storageEle)
    cmd = 'grep ^storage_path crab_' + crabTask.tag + '.cfg-Template'
    for file in os.popen(cmd).readlines():   # run command
        line        = file[:-1]              # strip '\n'
        # decode the storage directory name
        names       = line.split("=")        # splitting every '='
        names       = names[1:]
        storagePath = "=".join(names)
        storagePath = re.sub("\s", "",storagePath)
    cmd = 'grep ^user_remote_dir crab_' + crabTask.tag + '.cfg-Template'
    for file in os.popen(cmd).readlines():   # run command
        line        = file[:-1]              # strip '\n'
        # decode the storage directory name
        names        = line.split(" ")        # splitting every '='
        storagePath += names[-1]
    storageUrl = 'srm://' + storageEle + ':8443' + storagePath

    storagePath = adjustCfg(storagePath,subTask.nSubTaskLfn,tag)
    storageUrl  = adjustCfg(storageUrl, subTask.nSubTaskLfn,tag)
    
    print ' StorageUrl: ' + storageUrl
    
    # Create storage area
    if test == 0: # for testing we do not create the directories
        if (storageEle == 'srm.cern.ch'): 
            createDirCern(storagePath)
        else:
            createDirGeneral(storageEle,storagePath,fastCreate)

#    cmd  = "crab -create -debug 3 -USER.ui_working_dir=" + tag + " | tee forDaniele "
    cmd  = "crab -create -cfg crab_" + crabTask.tag + ".cfg -USER.ui_working_dir=" + tag
    print '  -> ' + cmd
    if test != 0:
        cmd = 'echo ' + cmd
    print '  ' + cmd
    for line in os.popen(cmd).readlines():   # run command
        line = line[:-1]                     # strip '\n'
        print ' CRAB: ' + line
        if re.search(pattern1,line):
            f = line.split("/")
            crabIdCheck = f[-2]
        if re.search(pattern2,line):
            f = line.split(" ")
            nJobsTotal = f[4]
            if nJobsTotal == '':
                nJobsTotal = f[2]
    # report
    print '  --> %s jobs created (%s).\n'%(nJobsTotal,tag)
    # and cleanup the temporary file for the subtask
    cmd = "rm -f " + mitDataset + ".lfns_" + tag
    os.system(cmd)

    # test something useful is going to happen
    if int(nJobsTotal) > 0:
        print ' More than zero (%d) jobs got created, go ahead and submit (%s).\n'%(int(nJobsTotal),crabId)
    else:
        print ' Zero or less jobs got created: cleanup and EXIT.'
        cmd = 'rm -rf `echo *' + crabId + '*`'
        print ' Cleanup: ' + cmd + '\n'
        #os.system(cmd)
        sys.exit(0)
        
    
    # adjust arguments
    cmd = 'input.py --db=' + lfnFile + '_' + tag + ' --option=xml --dataset=' + cmsDataset + \
          ' > ' + tag + '/share/arguments.xml'
    print '  update arguments: ' + cmd
    if test == 0:
        os.system(cmd)

    # loop through the file and determine the submission parameters
    block   = ''
    blocks  = []
    idx     = 0
    minIdxs = []
    maxIdxs = []
    
    fileInput = open(lfnFile + '_' + tag,'r')
    line = fileInput.readline()
    while (line != ''):
        line = line[:-1]
        if line[0] != '#':
            idx    += 1
            f       = line.split()
            block   = f[0]
            lfn     = f[1]
            nEvents = f[2]
            if len(blocks) == 0 or block != blocks[-1]:
                # new block found
                blocks .append(block)
                minIdxs.append(idx)
                maxIdxs.append(idx)
            else:
                maxIdxs[-1] = idx
        # keep the reading going
        line = fileInput.readline()
    fileInput .close()

    # merge blocks together if they use the same sites
    lastSites     = ''
    idx           = 0
    mergedIdx     = 0
    mergedBlocks  = []
    mergedSites   = []
    mergedMinIdxs = []
    mergedMaxIdxs = []
    print '\n Show the unmerged blocks:'
    for block in blocks:

        
        if fixSites != '':
            sites = fixSites
        else:
            if not os.path.exists(siteFile):
                cmd  = "sites.py --block=" + block + " --dbs=" + dbs
            else:
                cmd  = "cat " + siteFile + "| grep " + block + " | cut -d ':' -f2 | tr -d ' '"
                print 'CMD: ' + cmd
            for line in os.popen(cmd).readlines():   # run command
                line = line[:-1]
                sites = line

                sites = trans.translateSes(sites)
                sites = trans.selectPreferred()

        if sites == '':
            print ' ERROR - no sites for this data block, do not submit.'
        else:
            print '   Block ' + block + '  process:  %d  to  %d'%(minIdxs[idx],maxIdxs[idx]) + \
                  ' at\n        > ' + sites
        # block with different sites found
        if sites != lastSites:
            mergedSites  .append(sites)
            mergedBlocks .append(blocks [idx]) # only the first block with these sites is stored
            mergedMinIdxs.append(minIdxs[idx])
            mergedMaxIdxs.append(maxIdxs[idx])
            lastSites  = sites
            mergedIdx += 1
        else:
            mergedMaxIdxs[mergedIdx-1] = maxIdxs[idx]

        # last action in the loop: increment the unmerged blocks
        idx += 1

    # Show already what we will do
    idx = 0
    print '\n Show the merged blocks:'
    for block in mergedBlocks:
        print '   Merged Block ' + block + '  process:  %d  to  %d'\
              %(mergedMinIdxs[idx],mergedMaxIdxs[idx]) + \
              ' at\n        > ' + mergedSites[idx]
        # last action in the loop: increment the merged blocks
        idx += 1

    # perfrom the submission block by block (using the merged blocks of course)
    nSubmission = len(mergedBlocks)
    idx = 0
    print '\n Submit the merged blocks:'
    for block in mergedBlocks:
        print '   Merged Block ' + block + '  process:  %d  to  %d'\
              %(mergedMinIdxs[idx],mergedMaxIdxs[idx]) + \
              ' at\n        > ' + mergedSites[idx]
    
        nSubmit = '%d-%d'%(mergedMinIdxs[idx],mergedMaxIdxs[idx])
        if mergedMinIdxs[idx] == mergedMaxIdxs[idx]:
            nSubmit = '%d,%d'%(mergedMinIdxs[idx],100000000)
        cmd = 'crab -submit %s -continue %s -GRID.ce_white_list=%s'%(nSubmit,tag,mergedSites[idx])
        print '  ' + cmd + '\n'
        status = os.system(cmd)
        retry = False
        while status != 0 and not retry:
            retry = True
            print ' Submission failed  (%s) --> retry once!'%(cmd)
            status = os.system(cmd)

        # last action in the loop: increment the merged blocks
        idx += 1
    
    print '  Number of blocks submitted: %d' % nSubmission

# and cleanup the temporary file for the task
cmd = "rm -f crab_" + crabTask.tag + ".cfg crab_" + crabTask.tag + ".cfg-Template " \
      + cmsswPy + ' ' + cmsswPy + 'c'
os.system(cmd)

print ' Done... keep watching it...'
sys.exit(0)

# --------------------------------------------------------------------------------------------------
# Run a test job to test the configuration and measure the expected output size
# --------------------------------------------------------------------------------------------------
if noTestJob == 0:
    #-----------------------------------------------------------------------------------------------
    # use a specific file for test
    #-----------------------------------------------------------------------------------------------
    # first check whether we already have a local file
    print '\n Try to find an existing local file using "find"'
    f    = cmsDataset.split("/")
    name = f[1]
    vers = f[2]
    tier = f[3]
    file = ""
    lfn  = ""
    cmd  = 'find ./store/ -name ' + name
    print 'Searching: ' + cmd
    for line in os.popen(cmd).readlines():  # run command
        file = line[:-1]                    # strip '\n'

    # looks like there could be a file we found a directory, confirm
    if file != "":
        print '\n We have a directory ' + file + ' .. confirming'
        file = file + '/' + tier
        cmd = 'find ' + file + ' -name \*.root'
        for line in os.popen(cmd).readlines():  # run command
            if line.find(vers):
                file = line[:-1]                # strip '\n'
                lfn = file[1:]

        if os.path.exists(file):
            print ' moving on with locally found file: \n' + '   ' + file
        else:
            print ' no local file found'
            file = ""
            lfn = ""

    # now try to see whether we can find a file to download
    if file == "":
        cmd = './bin/findLfn.py --input=' + cmsDataset + ' | grep /store/'
        print '\n Find an LFN to download: ' + cmd
        for line in os.popen(cmd).readlines():  # run command
            if line.find("/store") != -1:
                lfn = line[:-1]                 # strip '\n'
                break
        if lfn == "":
            print "\n WARNING: No file found, continue assuming it is a simulation job.\n\n"
        else:
            print '  --> LFN: ' + lfn
            file = '.' + lfn

            if os.path.exists(file):
                print '  --> File already exists: ' + file
            else:
                cmd = './bin/downloadLfn.py ' + lfn
                print '  --> downloading: ' + cmd
                status = os.system(cmd)
                if status != 0:
                    print ' ERROR - failed to copy LFN. EXIT now!'
                    sys.exit(1)

    # Parse template input and adjust the input file to the newly copied lfn
    fileInput  = open(cmsswPy,'r')
    fileOutput = open("test-"+cmsswPy,'w')
    line = fileInput.readline()
    while (line != ''):
        if line.find("file:") != -1:
            line = '"file:' + lfn[1:] + '"\n'
        fileOutput.write(line)
        line = fileInput.readline()
    fileInput .close()
    fileOutput.close()

    # Setting the number of events (hard coded in the file so far)
    nTryEvts = 1000.

    print '\n  --> Please wait, running a test job now! Should be short (trying %.0f'%nTryEvts + \
          ' evts). Check log: cmssw.log'
    cmd = 'rm -f cmssw.log; /usr/bin/time --format "%e %U %S" cmsRun test-' + cmsswPy + \
          ' >& cmssw.log'
    print ' CMD: ' + cmd
    status = os.system(cmd)

    cmd = 'tail -1 cmssw.log'
    for file in os.popen(cmd).readlines():  # run command
        line       = file[:-1]              # strip '\n'
        f          = line.split()           # splitting every blank
        rtime      = float(f[0])            # wall clock time
        utime      = float(f[1])            # user time
        stime      = float(f[2])            # system time

    nEvtsTest = 1000
    cmd = 'grep \'Begin processing\' cmssw.log | tail -1'
    for file in os.popen(cmd).readlines():  # run command
        line       = file[:-1]              # strip '\n'
        # test whether there is a directory
        f          = line.split()           # splitting every blank
        nEvtsTest  = f[3]                   # this is the number of records processed
        nEvtsTest  = int(nEvtsTest[:-2])    # strip 'th'
        cmd = 'ls -s ' + mitDataset  + '*.root'
    size = 0
    for file in os.popen(cmd).readlines():  # run command
        line       = file[:-1]              # strip '\n'
        f          = line.split()           # splitting every blank
        size      += int(f[0])/1000.        # size in MB

    cmd = 'tail -1 cmssw.log'
    for file in os.popen(cmd).readlines():  # run command
        line       = file[:-1]              # strip '\n'
        # get total, user and system times
        names      = line.split()           # splitting every blank

    if nEvtsTest != nTryEvts:
        print ' WARNING - Instead of %f  did  %d'%(nTryEvts,nEvtsTest)

    print ' '
    print ' Number of test events produced:  %d'%nEvtsTest
    print ' File size for all events:        %.2f MB'%size
    print ' Processing time for all events:  %.2f secs (u: %.2f s: %.2f)'%(rtime,utime,stime)
    print ' '
    print '  --> 1 event == %.2f secs'%(rtime/nEvtsTest)
    print '  --> 1.00 GB == %d events'%(nEvtsTest/size*1024.)
    print '  --> %.2f GB == %d events'%(nevents/(nEvtsTest/size*1024.),nevents)
    print ' '

if testJob == 1:
    print '\n Test job finished, stopping now.\n'
    sys.exit(0)
Revision:	1.17
Committed:	Tue Feb 28 11:54:36 2012 UTC (13 years, 2 months ago) by paus
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	Mit_025e, Mit_025d
Changes since 1.16:	+104 -64 lines
Log Message:	Last updates.