CRAB/python/crab_util.py

###########################################################################
#
#   C O N V E N I E N C E    F U N C T I O N S
#
###########################################################################

import string, sys, os, time
import ConfigParser, re, popen2, select, fcntl

import common
from crab_exceptions import CrabException
from ServerConfig import *

###########################################################################
def parseOptions(argv):
    """
    Parses command-line options.
    Returns a dictionary with specified options as keys:
    -opt1             --> 'opt1' : None
    -opt2 val         --> 'opt2' : 'val'
    -opt3=val         --> 'opt3' : 'val'
    Usually called as
    options = parseOptions(sys.argv[1:])
    """
    options = {}
    argc = len(argv)
    i = 0
    while ( i < argc ):
        if argv[i][0] != '-':
            i = i + 1
            continue
        eq = string.find(argv[i], '=')
        if eq > 0 :
            opt = argv[i][:eq]
            val = argv[i][eq+1:]
            pass
        else:
            opt = argv[i]
            val = None
            if ( i+1 < argc and argv[i+1][0] != '-' ):
                i = i + 1
                val = argv[i]
                pass
            pass
        options[opt] = val
        i = i + 1
        pass
    return options

def loadConfig(file, config):
    """
    returns a dictionary with keys of the form
    <section>.<option> and the corresponding values
    """
    #config={}
    cp = ConfigParser.ConfigParser()
    cp.read(file)
    for sec in cp.sections():
        # print 'Section',sec
        for opt in cp.options(sec):
            #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
            config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
    return config

###########################################################################
def isInt(str):
    """ Is the given string an integer ?"""
    try: int(str)
    except ValueError: return 0
    return 1

###########################################################################
def isBool(str):
    """ Is the given string 0 or 1 ?"""
    if (str in ('0','1')): return 1
    return 0

###########################################################################
def parseRange(range):
    """
    Takes as the input a string with two integers separated by
    the minus sign and returns the tuple with these numbers:
    'n1-n2' -> (n1, n2)
    'n1'    -> (n1, n1)
    """
    start = None
    end   = None
    minus = string.find(range, '-')
    if ( minus < 0 ):
        if isInt(range):
            start = int(range)
            end   = start
            pass
        pass
    else:
        if isInt(range[:minus]) and isInt(range[minus+1:]):
            start = int(range[:minus])
            end   = int(range[minus+1:])
            pass
        pass
    return (start, end)

###########################################################################
def parseRange2(range):
    """
    Takes as the input a string in the form of a comma-separated
    numbers and ranges
    and returns a list with all specified numbers:
    'n1'          -> [n1]
    'n1-n2'       -> [n1, n1+1, ..., n2]
    'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
    """
    result = []
    if not range: return result

    comma = string.find(range, ',')
    if comma == -1: left = range
    else:           left = range[:comma]

    (n1, n2) = parseRange(left)
    while ( n1 <= n2 ):
        try:
            result.append(n1)
            n1 += 1
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    if comma != -1:
        try:
            result.extend(parseRange2(range[comma+1:]))
            pass
        except:
            msg = 'Syntax error in range <'+range+'>'
            raise CrabException(msg)

    return result

###########################################################################
def crabJobStatusToString(crab_status):
    """
    Convert one-letter crab job status into more readable string.
    """
    status={
           'C':'Created',
           'D':'Done',
           'H':'Hold',
           'U':'Ready',
           'I':'Scheduled',
           'X':'Canceled',
           'W':'Created',
           'R':'Running',
           'SC':'Checkpointed',
           'SS':'Scheduled',
           'SR':'Ready',
           'RE':'Ready',
           'SW':'Waiting',
           'SU':'Submitted',
           'S' :'Submitted (Boss)',
           'UN':'Undefined',
           'SK':'Cancelled',
           'SD':'Done (Success)',
           'SA':'Aborted',
           'DA':'Done (Aborted)',
           'SE':'Cleared',
           'OR':'Done (Success)',
           'A?':'Aborted',
           'K':'Killed',
           'E':'Cleared',
           'Z':'Cleared (Corrupt)',
           'NA':'Unknown',
           'I?':'Idle',
           'O?':'Done',
           'R?':'Running'
           }
    return status[crab_status]

###########################################################################
def findLastWorkDir(dir_prefix, where = None):

    if not where: where = os.getcwd() + '/'
    # dir_prefix usually has the form 'crab_0_'
    pattern = re.compile(dir_prefix)

    file_list = []
    for fl in os.listdir(where):
        if pattern.match(fl):
            file_list.append(fl)
            pass
        pass

    if len(file_list) == 0: return None

    file_list.sort()

    wdir = where + file_list[len(file_list)-1]
    return wdir

###########################################################################
def importName(module_name, name):
    """
    Import a named object from a Python module,
    i.e., it is an equivalent of 'from module_name import name'.
    """
    module = __import__(module_name, globals(), locals(), [name])
    return vars(module)[name]


###########################################################################

### WARNING This Function become USELESS after Boss API implementation
def runBossCommand(cmd, printout=0, timeout=3600):
    """
    Cd to correct directory before running a boss command
    """
    cwd = os.getcwd()
    os.chdir(common.work_space.shareDir())
    out = runCommand(cmd, printout, timeout)
    os.chdir(cwd)
    return out

###########################################################################
def readable(fd):
    return bool(select.select([fd], [], [], 0))

###########################################################################
def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
            fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.FNDELAY)

###########################################################################
def runCommand(cmd, printout=0, timeout=-1):
    """
    Run command 'cmd'.
    Returns command stdoutput+stderror string on success,
    or None if an error occurred.
    Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
    """

    if printout:
        common.logger.message(cmd)
    else:
        common.logger.debug(10,cmd)
        common.logger.write(cmd)
        pass

    child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = []
    errdata = []
    outeof = erreof = 0

    if timeout > 0 :
        maxwaittime = time.time() + timeout

    err = -1
    while (timeout == -1 or time.time() < maxwaittime):
        ready = select.select([outfd,errfd],[],[]) # wait for input
        if outfd in ready[0]:
            outchunk = outfile.read()
            if outchunk == '': outeof = 1
            outdata.append(outchunk)
        if errfd in ready[0]:
            errchunk = errfile.read()
            if errchunk == '': erreof = 1
            errdata.append(errchunk)
        if outeof and erreof:
            err = child.wait()
            break
        select.select([],[],[],.01) # give a little time for buffers to fill
    if err == -1:
        # kill the pid
        common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
        os.kill (child.pid, 9)
        err = child.wait()

    cmd_out = string.join(outdata,"")
    cmd_err = string.join(errdata,"")

    if err:
        common.logger.message('`'+cmd+'`\n   failed with exit code '
                           +`err`+'='+`(err&0xff)`+'(signal)+'
                           +`(err>>8)`+'(status)')
        common.logger.message(cmd_out)
        common.logger.message(cmd_err)
        return None

#    cmd_out = string.join(outdata,"")
#    cmd_err = string.join(errdata,"")
    cmd_out = cmd_out + cmd_err
    if printout:
        common.logger.message(cmd_out)
    else:
        common.logger.debug(10,cmd_out)
        common.logger.write(cmd_out)
        pass
    #print "<"+cmd_out+">"
    return cmd_out

####################################
def makeCksum(filename) :
    """
    make check sum using filename and content of file
    """

    from zlib import crc32
    hashString = filename

    inFile = open(filename, 'r')
    hashString += inFile.read()
    inFile.close()

    cksum = str(crc32(hashString))
    return cksum


def uniqueTaskName(nativeTaskName):
    """
    For task names that don't follow USER_crab_0_YYMMDD_HHMMSS, generate a new one that includes
    part of the internal BossLite taskId
    """

    import re
    taskName = "_".join(nativeTaskName.split('_')[:-1])
    reTask = re.compile('.*?_crab_0_\d{6}_\d{6}')
    isDefault = reTask.match(taskName)
    if not isDefault:
        taskName = "-".join(nativeTaskName.split('-')[:-4])

    return str(taskName)   # By default is unicode, ML needs standard string


def spanRanges(jobArray):
    """
    take array of job numbers and concatenate 1,2,3 to 1-3
    return string
    """

    output = ""
    jobArray.sort()

    previous = jobArray[0]-1
    for job in jobArray:
        if previous+1 == job:
            previous = job
            if len(output) > 0 :
                if output[-1] != "-":
                    output += "-"
            else :
                output += str(previous)
        else:
            output += str(previous) + "," + str(job)
            #output += "," + str(job)
            previous = job
    if len(jobArray) > 1 :
        output += str(previous)

    return output

def displayReport(self, header, lines, xml=''):

    counter = 0
    printline = ''
    printline+= header
    print printline
    print '---------------------------------------------------------------------------------------------------'

    for i in range(len(lines)):
        if counter != 0 and counter%10 == 0 :
            print '---------------------------------------------------------------------------------------------------'
        print lines[i]
        counter += 1
    if xml != '' :
        fileName = common.work_space.shareDir() + xml
        task = common._db.getTask()
        taskXML = common._db.serializeTask(task)
        common.logger.debug(5, taskXML)
        f = open(fileName, 'w')
        f.write(taskXML)
        f.close()
        pass

def CliServerParams(self):
    """
    Init client-server interactions
    """
    self.srvCfg = {}
    try:
        self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()

        self.server_name = str(self.srvCfg['serverName'])
        self.server_port = int(self.srvCfg['serverPort'])

        self.storage_name = str(self.srvCfg['storageName'])
        self.storage_path = str(self.srvCfg['storagePath'])
        self.storage_proto = str(self.srvCfg['storageProtocol'])
        self.storage_port = str(self.srvCfg['storagePort'])
    except KeyError:
        msg = 'No server selected or port specified.'
        msg = msg + 'Please specify a server in the crab cfg file'
        raise CrabException(msg)
        return

def bulkControl(self,list):
    """
    Check the BULK size and  reduce collection ...if needed
    """
    max_size = 400
    sub_bulk = []
    if len(list) > int(max_size):
        n_sub_bulk = int( int(len(list) ) / int(max_size) )
        for n in xrange(n_sub_bulk):
            first =n*int(max_size)
            last = (n+1)*int(max_size)
            sub_bulk.append(list[first:last])
        if len(list[last:-1]) < 50:
            for pp in list[last:-1]:
                sub_bulk[n_sub_bulk-1].append(pp)
        else:
            sub_bulk.append(list[last:])
    else:
        sub_bulk.append(list)

    return sub_bulk

def numberFile(file, txt):
    """
    append _'txt' before last extension of a file
    """
    txt=str(txt)
    p = string.split(file,".")
    # take away last extension
    name = p[0]
    for x in p[1:-1]:
        name=name+"."+x
    # add "_txt"
    if len(p)>1:
        ext = p[len(p)-1]
        result = name + '_' + txt + "." + ext
    else:
        result = name + '_' + txt

    return result

def readTXTfile(self,inFileName):
    """
    read file and return a list with the content
    """
    out_list=[]
    if os.path.exists(inFileName):
        f = open(inFileName, 'r')
        for line in  f.readlines():
            out_list.append(string.strip(line))
        f.close()
    else:
        msg = ' file '+str(inFileName)+' not found.'
        raise CrabException(msg)
    return out_list

def writeTXTfile(self, outFileName, args):
    """
    write a file with the given content ( args )
    """
    outFile = open(outFileName,"a")
    outFile.write(str(args))
    outFile.close()
    return

def getLocalDomain(self):
    """
     Get local domain name
    """
    import socket
    tmp=socket.gethostname()
    dot=string.find(tmp,'.')
    if (dot==-1):
        msg='Unkown domain name. Cannot use local scheduler'
        raise CrabException(msg)
    localDomainName = string.split(tmp,'.',1)[-1]
    return localDomainName

####################################
if __name__ == '__main__':
    print 'sys.argv[1] =',sys.argv[1]
    list = parseRange2(sys.argv[1])
    print list
    cksum = makeCksum("crab_util.py")
    print cksum

Revision:	1.53
Committed:	Wed Aug 20 11:38:50 2008 UTC (16 years, 8 months ago) by mcinquil
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_3_2_pre5, CRAB_2_3_2_pre4, CRAB_2_3_2_pre3
Changes since 1.52:	+2 -2 lines
Log Message:	Fixed bug on last job lost for more then 1 collection + added usage of xrange
#	User	Rev	Content
1	nsmirnov	1.1	###########################################################################
2			#
3			# C O N V E N I E N C E F U N C T I O N S
4			#
5			###########################################################################
6
7	slacapra	1.12	import string, sys, os, time
8	slacapra	1.16	import ConfigParser, re, popen2, select, fcntl
9	nsmirnov	1.1
10	nsmirnov	1.2	import common
11	slacapra	1.30	from crab_exceptions import CrabException
12	spiga	1.42	from ServerConfig import *
13	nsmirnov	1.1
14			###########################################################################
15			def parseOptions(argv):
16			"""
17			Parses command-line options.
18			Returns a dictionary with specified options as keys:
19			-opt1 --> 'opt1' : None
20			-opt2 val --> 'opt2' : 'val'
21			-opt3=val --> 'opt3' : 'val'
22			Usually called as
23			options = parseOptions(sys.argv[1:])
24			"""
25			options = {}
26			argc = len(argv)
27			i = 0
28			while ( i < argc ):
29			if argv[i][0] != '-':
30			i = i + 1
31			continue
32			eq = string.find(argv[i], '=')
33			if eq > 0 :
34			opt = argv[i][:eq]
35			val = argv[i][eq+1:]
36			pass
37			else:
38			opt = argv[i]
39			val = None
40			if ( i+1 < argc and argv[i+1][0] != '-' ):
41			i = i + 1
42			val = argv[i]
43			pass
44			pass
45			options[opt] = val
46			i = i + 1
47			pass
48			return options
49
50	slacapra	1.47	def loadConfig(file, config):
51	nsmirnov	1.1	"""
52			returns a dictionary with keys of the form
53			<section>.<option> and the corresponding values
54			"""
55	slacapra	1.47	#config={}
56	nsmirnov	1.1	cp = ConfigParser.ConfigParser()
57			cp.read(file)
58			for sec in cp.sections():
59			# print 'Section',sec
60			for opt in cp.options(sec):
61			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
62			config[sec+'.'+opt] = string.strip(cp.get(sec,opt))
63			return config
64
65			###########################################################################
66			def isInt(str):
67			""" Is the given string an integer ?"""
68			try: int(str)
69			except ValueError: return 0
70			return 1
71
72			###########################################################################
73			def isBool(str):
74			""" Is the given string 0 or 1 ?"""
75			if (str in ('0','1')): return 1
76			return 0
77
78			###########################################################################
79	nsmirnov	1.3	def parseRange(range):
80			"""
81			Takes as the input a string with two integers separated by
82			the minus sign and returns the tuple with these numbers:
83			'n1-n2' -> (n1, n2)
84			'n1' -> (n1, n1)
85			"""
86			start = None
87			end = None
88			minus = string.find(range, '-')
89			if ( minus < 0 ):
90			if isInt(range):
91			start = int(range)
92			end = start
93			pass
94			pass
95			else:
96			if isInt(range[:minus]) and isInt(range[minus+1:]):
97			start = int(range[:minus])
98			end = int(range[minus+1:])
99			pass
100			pass
101			return (start, end)
102
103			###########################################################################
104	nsmirnov	1.4	def parseRange2(range):
105			"""
106			Takes as the input a string in the form of a comma-separated
107			numbers and ranges
108			and returns a list with all specified numbers:
109			'n1' -> [n1]
110			'n1-n2' -> [n1, n1+1, ..., n2]
111			'n1,n2-n3,n4' -> [n1, n2, n2+1, ..., n3, n4]
112			"""
113	slacapra	1.30	result = []
114			if not range: return result
115	nsmirnov	1.4
116			comma = string.find(range, ',')
117			if comma == -1: left = range
118			else: left = range[:comma]
119
120			(n1, n2) = parseRange(left)
121			while ( n1 <= n2 ):
122	slacapra	1.11	try:
123	slacapra	1.30	result.append(n1)
124	slacapra	1.11	n1 += 1
125			pass
126			except:
127			msg = 'Syntax error in range <'+range+'>'
128			raise CrabException(msg)
129	nsmirnov	1.4
130			if comma != -1:
131	slacapra	1.11	try:
132	slacapra	1.30	result.extend(parseRange2(range[comma+1:]))
133	slacapra	1.11	pass
134			except:
135			msg = 'Syntax error in range <'+range+'>'
136			raise CrabException(msg)
137	nsmirnov	1.4
138	slacapra	1.30	return result
139	nsmirnov	1.4
140			###########################################################################
141	nsmirnov	1.3	def crabJobStatusToString(crab_status):
142			"""
143			Convert one-letter crab job status into more readable string.
144			"""
145	corvo	1.27	status={
146	slacapra	1.38	'C':'Created',
147			'D':'Done',
148	corvo	1.27	'H':'Hold',
149			'U':'Ready',
150			'I':'Scheduled',
151			'X':'Canceled',
152			'W':'Created',
153			'R':'Running',
154			'SC':'Checkpointed',
155			'SS':'Scheduled',
156			'SR':'Ready',
157			'RE':'Ready',
158			'SW':'Waiting',
159			'SU':'Submitted',
160			'S' :'Submitted (Boss)',
161			'UN':'Undefined',
162			'SK':'Cancelled',
163			'SD':'Done (Success)',
164			'SA':'Aborted',
165			'DA':'Done (Aborted)',
166			'SE':'Cleared',
167			'OR':'Done (Success)',
168			'A?':'Aborted',
169			'K':'Killed',
170			'E':'Cleared',
171			'Z':'Cleared (Corrupt)',
172			'NA':'Unknown',
173			'I?':'Idle',
174			'O?':'Done',
175			'R?':'Running'
176			}
177			return status[crab_status]
178	nsmirnov	1.3
179			###########################################################################
180	nsmirnov	1.1	def findLastWorkDir(dir_prefix, where = None):
181
182			if not where: where = os.getcwd() + '/'
183			# dir_prefix usually has the form 'crab_0_'
184			pattern = re.compile(dir_prefix)
185
186			file_list = []
187			for fl in os.listdir(where):
188			if pattern.match(fl):
189			file_list.append(fl)
190			pass
191			pass
192
193			if len(file_list) == 0: return None
194
195			file_list.sort()
196
197			wdir = where + file_list[len(file_list)-1]
198			return wdir
199
200			###########################################################################
201			def importName(module_name, name):
202			"""
203			Import a named object from a Python module,
204			i.e., it is an equivalent of 'from module_name import name'.
205			"""
206			module = __import__(module_name, globals(), locals(), [name])
207			return vars(module)[name]
208
209	spiga	1.9
210	slacapra	1.11	###########################################################################
211	spiga	1.28
212	slacapra	1.29	### WARNING This Function become USELESS after Boss API implementation
213	gutsche	1.24	def runBossCommand(cmd, printout=0, timeout=3600):
214	slacapra	1.11	"""
215			Cd to correct directory before running a boss command
216			"""
217			cwd = os.getcwd()
218			os.chdir(common.work_space.shareDir())
219			out = runCommand(cmd, printout, timeout)
220			os.chdir(cwd)
221			return out
222	spiga	1.9
223	nsmirnov	1.1	###########################################################################
224	slacapra	1.16	def readable(fd):
225	ewv	1.44	return bool(select.select([fd], [], [], 0))
226	slacapra	1.16
227			###########################################################################
228			def makeNonBlocking(fd):
229			fl = fcntl.fcntl(fd, fcntl.F_GETFL)
230			try:
231			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.O_NDELAY)
232			except AttributeError:
233			fcntl.fcntl(fd, fcntl.F_SETFL, fl \| os.FNDELAY)
234
235			###########################################################################
236	slacapra	1.11	def runCommand(cmd, printout=0, timeout=-1):
237	nsmirnov	1.1	"""
238			Run command 'cmd'.
239			Returns command stdoutput+stderror string on success,
240			or None if an error occurred.
241	slacapra	1.16	Following recipe on http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
242	nsmirnov	1.1	"""
243	fanzago	1.15
244	slacapra	1.11	if printout:
245			common.logger.message(cmd)
246			else:
247	slacapra	1.13	common.logger.debug(10,cmd)
248	slacapra	1.11	common.logger.write(cmd)
249	slacapra	1.29	pass
250	slacapra	1.11
251	slacapra	1.16	child = popen2.Popen3(cmd, 1) # capture stdout and stderr from command
252			child.tochild.close() # don't need to talk to child
253	ewv	1.44	outfile = child.fromchild
254	slacapra	1.16	outfd = outfile.fileno()
255			errfile = child.childerr
256			errfd = errfile.fileno()
257			makeNonBlocking(outfd) # don't deadlock!
258			makeNonBlocking(errfd)
259			outdata = []
260			errdata = []
261			outeof = erreof = 0
262
263			if timeout > 0 :
264			maxwaittime = time.time() + timeout
265
266			err = -1
267			while (timeout == -1 or time.time() < maxwaittime):
268			ready = select.select([outfd,errfd],[],[]) # wait for input
269			if outfd in ready[0]:
270			outchunk = outfile.read()
271			if outchunk == '': outeof = 1
272			outdata.append(outchunk)
273			if errfd in ready[0]:
274			errchunk = errfile.read()
275			if errchunk == '': erreof = 1
276			errdata.append(errchunk)
277			if outeof and erreof:
278			err = child.wait()
279			break
280	slacapra	1.21	select.select([],[],[],.01) # give a little time for buffers to fill
281	slacapra	1.16	if err == -1:
282			# kill the pid
283	slacapra	1.18	common.logger.message('killing process '+(cmd)+' with timeout '+str(timeout))
284	slacapra	1.16	os.kill (child.pid, 9)
285	slacapra	1.11	err = child.wait()
286	slacapra	1.16
287	fanzago	1.17	cmd_out = string.join(outdata,"")
288			cmd_err = string.join(errdata,"")
289
290	slacapra	1.11	if err:
291			common.logger.message('`'+cmd+'`\n failed with exit code '
292			+`err`+'='+`(err&0xff)`+'(signal)+'
293			+`(err>>8)`+'(status)')
294			common.logger.message(cmd_out)
295			common.logger.message(cmd_err)
296	nsmirnov	1.1	return None
297
298	fanzago	1.17	# cmd_out = string.join(outdata,"")
299			# cmd_err = string.join(errdata,"")
300	nsmirnov	1.1	cmd_out = cmd_out + cmd_err
301	slacapra	1.11	if printout:
302			common.logger.message(cmd_out)
303			else:
304	slacapra	1.13	common.logger.debug(10,cmd_out)
305	slacapra	1.11	common.logger.write(cmd_out)
306	slacapra	1.29	pass
307	slacapra	1.16	#print "<"+cmd_out+">"
308	nsmirnov	1.1	return cmd_out
309	nsmirnov	1.4
310	slacapra	1.11	####################################
311	gutsche	1.20	def makeCksum(filename) :
312			"""
313	ewv	1.44	make check sum using filename and content of file
314	gutsche	1.20	"""
315
316	ewv	1.44	from zlib import crc32
317			hashString = filename
318	gutsche	1.20
319	ewv	1.44	inFile = open(filename, 'r')
320			hashString += inFile.read()
321			inFile.close()
322	gutsche	1.20
323	ewv	1.44	cksum = str(crc32(hashString))
324	gutsche	1.20	return cksum
325
326	ewv	1.52
327			def uniqueTaskName(nativeTaskName):
328			"""
329			For task names that don't follow USER_crab_0_YYMMDD_HHMMSS, generate a new one that includes
330			part of the internal BossLite taskId
331			"""
332
333			import re
334			taskName = "_".join(nativeTaskName.split('_')[:-1])
335			reTask = re.compile('.*?_crab_0_\d{6}_\d{6}')
336			isDefault = reTask.match(taskName)
337			if not isDefault:
338			taskName = "-".join(nativeTaskName.split('-')[:-4])
339
340			return str(taskName) # By default is unicode, ML needs standard string
341
342
343	gutsche	1.32	def spanRanges(jobArray):
344			"""
345			take array of job numbers and concatenate 1,2,3 to 1-3
346			return string
347			"""
348
349			output = ""
350	mcinquil	1.35	jobArray.sort()
351	ewv	1.44
352	gutsche	1.32	previous = jobArray[0]-1
353			for job in jobArray:
354			if previous+1 == job:
355			previous = job
356			if len(output) > 0 :
357			if output[-1] != "-":
358			output += "-"
359			else :
360			output += str(previous)
361			else:
362			output += str(previous) + "," + str(job)
363	mcinquil	1.35	#output += "," + str(job)
364	gutsche	1.32	previous = job
365			if len(jobArray) > 1 :
366			output += str(previous)
367
368			return output
369
370	spiga	1.40	def displayReport(self, header, lines, xml=''):
371	ewv	1.44
372	spiga	1.49	counter = 0
373			printline = ''
374			printline+= header
375			print printline
376			print '---------------------------------------------------------------------------------------------------'
377
378			for i in range(len(lines)):
379			if counter != 0 and counter%10 == 0 :
380			print '---------------------------------------------------------------------------------------------------'
381			print lines[i]
382			counter += 1
383			if xml != '' :
384	spiga	1.40	fileName = common.work_space.shareDir() + xml
385			task = common._db.getTask()
386			taskXML = common._db.serializeTask(task)
387			common.logger.debug(5, taskXML)
388			f = open(fileName, 'w')
389			f.write(taskXML)
390			f.close()
391			pass
392	spiga	1.39
393	spiga	1.42	def CliServerParams(self):
394	slacapra	1.45	"""
395			Init client-server interactions
396			"""
397			self.srvCfg = {}
398			try:
399			self.srvCfg = ServerConfig(self.cfg_params['CRAB.server_name']).config()
400
401			self.server_name = str(self.srvCfg['serverName'])
402			self.server_port = int(self.srvCfg['serverPort'])
403
404			self.storage_name = str(self.srvCfg['storageName'])
405			self.storage_path = str(self.srvCfg['storagePath'])
406			self.storage_proto = str(self.srvCfg['storageProtocol'])
407			self.storage_port = str(self.srvCfg['storagePort'])
408			except KeyError:
409			msg = 'No server selected or port specified.'
410			msg = msg + 'Please specify a server in the crab cfg file'
411			raise CrabException(msg)
412	spiga	1.42	return
413	spiga	1.39
414	ewv	1.44	def bulkControl(self,list):
415	slacapra	1.45	"""
416			Check the BULK size and reduce collection ...if needed
417			"""
418			max_size = 400
419			sub_bulk = []
420			if len(list) > int(max_size):
421			n_sub_bulk = int( int(len(list) ) / int(max_size) )
422	mcinquil	1.53	for n in xrange(n_sub_bulk):
423	slacapra	1.45	first =n*int(max_size)
424			last = (n+1)*int(max_size)
425			sub_bulk.append(list[first:last])
426			if len(list[last:-1]) < 50:
427			for pp in list[last:-1]:
428			sub_bulk[n_sub_bulk-1].append(pp)
429	spiga	1.43	else:
430	mcinquil	1.53	sub_bulk.append(list[last:])
431	slacapra	1.45	else:
432			sub_bulk.append(list)
433
434			return sub_bulk
435
436			def numberFile(file, txt):
437			"""
438			append _'txt' before last extension of a file
439			"""
440			txt=str(txt)
441			p = string.split(file,".")
442			# take away last extension
443			name = p[0]
444			for x in p[1:-1]:
445			name=name+"."+x
446			# add "_txt"
447			if len(p)>1:
448			ext = p[len(p)-1]
449			result = name + '_' + txt + "." + ext
450			else:
451			result = name + '_' + txt
452	spiga	1.43
453	slacapra	1.45	return result
454	spiga	1.46
455			def readTXTfile(self,inFileName):
456			"""
457			read file and return a list with the content
458			"""
459			out_list=[]
460			if os.path.exists(inFileName):
461			f = open(inFileName, 'r')
462			for line in f.readlines():
463	ewv	1.52	out_list.append(string.strip(line))
464	spiga	1.46	f.close()
465			else:
466			msg = ' file '+str(inFileName)+' not found.'
467	ewv	1.52	raise CrabException(msg)
468	spiga	1.46	return out_list
469
470			def writeTXTfile(self, outFileName, args):
471			"""
472			write a file with the given content ( args )
473			"""
474			outFile = open(outFileName,"a")
475			outFile.write(str(args))
476			outFile.close()
477			return
478
479	spiga	1.51	def getLocalDomain(self):
480	ewv	1.52	"""
481	spiga	1.51	Get local domain name
482	ewv	1.52	"""
483	spiga	1.51	import socket
484			tmp=socket.gethostname()
485			dot=string.find(tmp,'.')
486			if (dot==-1):
487			msg='Unkown domain name. Cannot use local scheduler'
488			raise CrabException(msg)
489			localDomainName = string.split(tmp,'.',1)[-1]
490	ewv	1.52	return localDomainName
491	spiga	1.51
492	gutsche	1.20	####################################
493	nsmirnov	1.4	if __name__ == '__main__':
494			print 'sys.argv[1] =',sys.argv[1]
495			list = parseRange2(sys.argv[1])
496			print list
497	slacapra	1.29	cksum = makeCksum("crab_util.py")
498			print cksum
499	ewv	1.44