| 1 |
#!/usr/bin/env python
|
| 2 |
import os, string, re
|
| 3 |
import common
|
| 4 |
from DLSInfo import *
|
| 5 |
|
| 6 |
# ####################################
|
| 7 |
class DataLocationError(exceptions.Exception):
|
| 8 |
def __init__(self, errorMessage):
|
| 9 |
self.args=errorMessage
|
| 10 |
exceptions.Exception.__init__(self, self.args)
|
| 11 |
pass
|
| 12 |
|
| 13 |
def getErrorMessage(self):
|
| 14 |
""" Return exception error """
|
| 15 |
return "%s" % (self.args)
|
| 16 |
|
| 17 |
# ####################################
|
| 18 |
# class to extact data location
|
| 19 |
class DataLocation:
|
| 20 |
def __init__(self, Listfileblocks, cfg_params):
|
| 21 |
|
| 22 |
# Attributes
|
| 23 |
self.Listfileblocks = Listfileblocks # DLS input: list of fileblocks lists
|
| 24 |
|
| 25 |
self.cfg_params = cfg_params
|
| 26 |
|
| 27 |
self.SelectedSites = {} # DLS output: list of sites hosting fileblocks
|
| 28 |
# retrieved using method getSites
|
| 29 |
|
| 30 |
# #######################################################################
|
| 31 |
def fetchDLSInfo(self):
|
| 32 |
"""
|
| 33 |
Contact DLS
|
| 34 |
"""
|
| 35 |
dlstype=''
|
| 36 |
|
| 37 |
useDBS = self.cfg_params.get('CMSSW.dbs_url',None)
|
| 38 |
scheduler = self.cfg_params.get('CRAB.scheduler',None).lower()
|
| 39 |
global_url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
|
| 40 |
|
| 41 |
# Force show_prod=1 for everybody, grid jobs rely on central black list and t1access role to limit access to T1's
|
| 42 |
|
| 43 |
# Don't switch to DBS just because user specified the global URL in config
|
| 44 |
|
| 45 |
self.cfg_params['CMSSW.show_prod'] = 1
|
| 46 |
if useDBS == global_url:
|
| 47 |
useDBS = None
|
| 48 |
|
| 49 |
if useDBS:
|
| 50 |
dlstype='dbs'
|
| 51 |
DLS_type="DLS_TYPE_%s"%dlstype.upper()
|
| 52 |
dls=DLSInfo(DLS_type,self.cfg_params)
|
| 53 |
blockSites = self.PrepareDict(dls)
|
| 54 |
else:
|
| 55 |
dlstype='phedex'
|
| 56 |
DLS_type="DLS_TYPE_%s"%dlstype.upper()
|
| 57 |
dls=DLSInfo(DLS_type,self.cfg_params)
|
| 58 |
blockSites = dls.getReplicasBulk(self.Listfileblocks)
|
| 59 |
|
| 60 |
self.SelectedSites = blockSites
|
| 61 |
|
| 62 |
def PrepareDict(self,dls):
|
| 63 |
## find the replicas for each block
|
| 64 |
failCount = 0
|
| 65 |
countblock=0
|
| 66 |
blockSites = {}
|
| 67 |
for fileblocks in self.Listfileblocks:
|
| 68 |
countblock=countblock+1
|
| 69 |
try:
|
| 70 |
replicas=dls.getReplicas(fileblocks)
|
| 71 |
common.logger.debug("sites are %s"%replicas)
|
| 72 |
if len(replicas)!=0:
|
| 73 |
blockSites[fileblocks] = replicas
|
| 74 |
else:
|
| 75 |
# add empty entry if no replicas found
|
| 76 |
blockSites[fileblocks] = ''
|
| 77 |
|
| 78 |
except DLSNoReplicas, ex:
|
| 79 |
common.logger.debug(str(ex.getErrorMessage()))
|
| 80 |
common.logger.debug("Block not hosted by any site, continuing.\n")
|
| 81 |
blockSites[fileblocks] = ''
|
| 82 |
failCount = failCount + 1
|
| 83 |
except:
|
| 84 |
raise DataLocationError('')
|
| 85 |
|
| 86 |
if countblock == failCount:
|
| 87 |
msg = "All data blocks encountered a DLS error. Quitting."
|
| 88 |
raise DataLocationError(msg)
|
| 89 |
return blockSites
|
| 90 |
# #######################################################################
|
| 91 |
def getSites(self):
|
| 92 |
"""
|
| 93 |
get the sites hosting all the needed data
|
| 94 |
"""
|
| 95 |
return self.SelectedSites
|
| 96 |
|
| 97 |
#######################################################################
|
| 98 |
def uniquelist(self, old):
|
| 99 |
"""
|
| 100 |
remove duplicates from a list
|
| 101 |
"""
|
| 102 |
nd={}
|
| 103 |
for e in old:
|
| 104 |
nd[e]=0
|
| 105 |
return nd.keys()
|
