CRAB/python/multicrab.py

#!/usr/bin/env python
import sys, os, time, string, shutil
from crab_util import *
from crab import *
import common

###########################################################################
class MultiCrab:
    def __init__(self, opts):
        self.prog_name='multicrab'
        # Configuration file
        self.cfg_fname = None
        # Continuation flag
        self.flag_continue = 0
        self.continue_dir = None
        self.processContinueOption_(opts)

        self.processIniFile_(opts)
        # configuration
        self.opts=opts

        if not self.flag_continue:
            self.createWorkSpace()

        print self.prog_name + ' running on ' +  time.ctime(time.time())
        print '  working directory   ' + self.continue_dir

        crabs=[]

    def processContinueOption_(self,opts):

        # Look for the '-continue' option.

        for opt in opts.keys():
            if ( opt in ('-continue','-c') ):
                self.flag_continue = 1
                val = opts[opt]
                if val:
                    if val[0] == '/': self.continue_dir = val     # abs path
                    else: self.continue_dir = os.getcwd() + '/' + val      # rel path
                    pass
                break
            pass

        # Look for actions which has sense only with '-continue'

        if "-create" not in opts.keys() :
            self.flag_continue = 1

        if not self.flag_continue: return

        if not self.continue_dir:
            prefix = self.prog_name + '_'
            self.continue_dir = findLastWorkDir(prefix)
            pass

        if not self.continue_dir:
            raise CrabException('Cannot find last working directory.')

        if not os.path.exists(self.continue_dir):
            msg = 'Cannot continue because the working directory <'
            msg += self.continue_dir
            msg += '> does not exist.'
            raise CrabException(msg)

        return

    def createWorkSpace(self):
        # create WorkingDir for Multicrab
        if 'MULTICRAB.working_dir' in self.opts.keys():    
            self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
        else:
            current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
            self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time

        if self.continue_dir and not os.path.exists(self.continue_dir):
            try:
                os.mkdir(self.continue_dir)
            except OSError:
                msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        else:
            msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
            raise CrabException(msg)

        shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
        
        return
        
    def processIniFile_(self, opts):
        """
        Processes a configuration INI-file.
        """

        # Extract cfg-file name from the cmd-line options.

        for opt in opts.keys():
            if ( opt == '-cfg' ):
                if self.flag_continue:
                    raise CrabException('-continue and -cfg cannot coexist.')
                if opts[opt] : self.cfg_fname = opts[opt]
                else : processHelpOptions()
                pass
            pass

        # Set default cfg-fname

        if self.cfg_fname == None:
            if self.flag_continue:
                self.cfg_fname = self.continue_dir + '/multicrab.cfg'
            else:
                self.cfg_fname = 'multicrab.cfg'
                pass
            pass

        # Load cfg-file

        if self.cfg_fname != None:
            if os.path.exists(self.cfg_fname):
                self.cfg_params = self.loadMultiConfig(self.cfg_fname)
                pass
            else:
                msg = 'cfg-file '+self.cfg_fname+' not found.'
                raise CrabException(msg)
                pass
            pass

        # process the [CRAB] section

        lhp = len('MULTICRAB.')
        for k in self.cfg_params.keys():
            if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
                opt = '-'+k[lhp:]
                if len(opt) >= 3 and opt[:3] == '-__': continue
                if opt not in opts.keys():
                    opts[opt] = self.cfg_params[k]
                    pass
                pass
            pass

        self.cfg_params_dataset = {}
        common_opts = {}
        # first get common sections
        for sec in self.cfg_params:
            if sec in ['MULTICRAB']:
                cfg_common=self.cfg_params[sec]
                continue
            if sec in ['COMMON']:
                common_opts=self.cfg_params[sec]
                continue
            pass

        # then Dataset's specific
        for sec in self.cfg_params:
            if sec in ['MULTICRAB', 'COMMON']: continue
            self.cfg_params_dataset[sec]=self.cfg_params[sec]
            # add common to all dataset
            for key in common_opts:
                self.cfg_params_dataset[sec][key]=common_opts[key]
            pass

        self.cfg=cfg_common['cfg']

        # read crab.cfg file and search for storage_path
        cfg_params = loadConfig(self.cfg,{})
        self.user_remote_dir = cfg_params.get("USER.user_remote_dir",None)
        return

    def loadMultiConfig(self, file):
        """
        returns a dictionary with keys of the form
        <section>.<option> and the corresponding values
        """
        config={}
        cp = ConfigParser.ConfigParser()
        cp.read(file)
        for sec in cp.sections():
            # print 'Section',sec
            config[sec]={}
            for opt in cp.options(sec):
                #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
                config[sec][opt] = string.strip(cp.get(sec,opt))
        return config

    def run(self):
        #run crabs
        for sec in self.cfg_params_dataset:
            options={}
            if self.flag_continue:
                options['-c']=sec
            # DatasetName to be used
            options['-USER.ui_working_dir']=sec
            # options from multicrab.cfg
            for opt in self.cfg_params_dataset[sec]:
                tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
                options[tmp]=self.cfg_params_dataset[sec][opt]
            # add section to storage_path if exist in crab.cfg
            if self.user_remote_dir:
                options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
            # Input options (command)
            for opt in self.opts:
                options[opt]=self.opts[opt]
            try:
                #print options
                crab = Crab(options)
                crab.run()
                common.apmon.free()
                del crab
            except CrabException, e:
                print '\n' + common.prog_name + ': ' + str(e) + '\n'
                if common.logger:
                    common.logger.write('ERROR: '+str(e)+'\n')
                    pass
                pass
            pass
        pass
        
        #common.apmon.free()

###########################################################################
if __name__ == '__main__':
    ## Get rid of some useless warning
    try:
        import warnings
        warnings.simplefilter("ignore", RuntimeWarning)
    except ImportError:
        pass # too bad, you'll get the warning

    # Parse command-line options and create a dictionary with
    # key-value pairs.
    options = parseOptions(sys.argv[1:])

    # Process "help" options, such as '-help', '-version'
    if processHelpOptions(options) : sys.exit(0)

    # Create, initialize, and run a Crab object
    try:
        multicrab = MultiCrab(options)
        multicrab.run()
    except CrabException, e:
        print '\n' + common.prog_name + ': ' + str(e) + '\n'

    pass
Revision:	1.5
Committed:	Tue Dec 2 10:48:41 2008 UTC (16 years, 5 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_4_3_pre5
Changes since 1.4:	+4 -3 lines
Log Message:	fix problem with user_remote_dir
#	User	Rev	Content
1	slacapra	1.1	#!/usr/bin/env python
2			import sys, os, time, string, shutil
3			from crab_util import *
4			from crab import *
5			import common
6
7			###########################################################################
8			class MultiCrab:
9			def __init__(self, opts):
10			self.prog_name='multicrab'
11			# Configuration file
12			self.cfg_fname = None
13			# Continuation flag
14			self.flag_continue = 0
15			self.continue_dir = None
16			self.processContinueOption_(opts)
17
18			self.processIniFile_(opts)
19			# configuration
20			self.opts=opts
21
22			if not self.flag_continue:
23			self.createWorkSpace()
24
25			print self.prog_name + ' running on ' + time.ctime(time.time())
26			print ' working directory ' + self.continue_dir
27
28			crabs=[]
29
30			def processContinueOption_(self,opts):
31
32			# Look for the '-continue' option.
33
34			for opt in opts.keys():
35			if ( opt in ('-continue','-c') ):
36			self.flag_continue = 1
37			val = opts[opt]
38			if val:
39			if val[0] == '/': self.continue_dir = val # abs path
40			else: self.continue_dir = os.getcwd() + '/' + val # rel path
41			pass
42			break
43			pass
44
45			# Look for actions which has sense only with '-continue'
46
47			if "-create" not in opts.keys() :
48			self.flag_continue = 1
49
50			if not self.flag_continue: return
51
52			if not self.continue_dir:
53			prefix = self.prog_name + '_'
54			self.continue_dir = findLastWorkDir(prefix)
55			pass
56
57			if not self.continue_dir:
58			raise CrabException('Cannot find last working directory.')
59
60			if not os.path.exists(self.continue_dir):
61			msg = 'Cannot continue because the working directory <'
62			msg += self.continue_dir
63			msg += '> does not exist.'
64			raise CrabException(msg)
65
66			return
67
68			def createWorkSpace(self):
69			# create WorkingDir for Multicrab
70			if 'MULTICRAB.working_dir' in self.opts.keys():
71			self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
72			else:
73			current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
74			self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time
75
76			if self.continue_dir and not os.path.exists(self.continue_dir):
77			try:
78			os.mkdir(self.continue_dir)
79			except OSError:
80			msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
81			raise CrabException(msg)
82			pass
83			else:
84			msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
85			raise CrabException(msg)
86
87			shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
88
89			return
90
91			def processIniFile_(self, opts):
92			"""
93			Processes a configuration INI-file.
94			"""
95
96			# Extract cfg-file name from the cmd-line options.
97
98			for opt in opts.keys():
99			if ( opt == '-cfg' ):
100			if self.flag_continue:
101			raise CrabException('-continue and -cfg cannot coexist.')
102			if opts[opt] : self.cfg_fname = opts[opt]
103			else : processHelpOptions()
104			pass
105			pass
106
107			# Set default cfg-fname
108
109			if self.cfg_fname == None:
110			if self.flag_continue:
111			self.cfg_fname = self.continue_dir + '/multicrab.cfg'
112			else:
113			self.cfg_fname = 'multicrab.cfg'
114			pass
115			pass
116
117			# Load cfg-file
118
119			if self.cfg_fname != None:
120			if os.path.exists(self.cfg_fname):
121			self.cfg_params = self.loadMultiConfig(self.cfg_fname)
122			pass
123			else:
124			msg = 'cfg-file '+self.cfg_fname+' not found.'
125			raise CrabException(msg)
126			pass
127			pass
128
129			# process the [CRAB] section
130
131			lhp = len('MULTICRAB.')
132			for k in self.cfg_params.keys():
133			if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
134			opt = '-'+k[lhp:]
135			if len(opt) >= 3 and opt[:3] == '-__': continue
136			if opt not in opts.keys():
137			opts[opt] = self.cfg_params[k]
138			pass
139			pass
140			pass
141
142			self.cfg_params_dataset = {}
143	slacapra	1.3	common_opts = {}
144	slacapra	1.1	# first get common sections
145			for sec in self.cfg_params:
146			if sec in ['MULTICRAB']:
147			cfg_common=self.cfg_params[sec]
148			continue
149			if sec in ['COMMON']:
150	slacapra	1.3	common_opts=self.cfg_params[sec]
151	slacapra	1.1	continue
152			pass
153
154			# then Dataset's specific
155			for sec in self.cfg_params:
156			if sec in ['MULTICRAB', 'COMMON']: continue
157			self.cfg_params_dataset[sec]=self.cfg_params[sec]
158			# add common to all dataset
159	slacapra	1.3	for key in common_opts:
160			self.cfg_params_dataset[sec][key]=common_opts[key]
161	slacapra	1.1	pass
162
163			self.cfg=cfg_common['cfg']
164
165	slacapra	1.2	# read crab.cfg file and search for storage_path
166			cfg_params = loadConfig(self.cfg,{})
167	slacapra	1.5	self.user_remote_dir = cfg_params.get("USER.user_remote_dir",None)
168	slacapra	1.1	return
169
170			def loadMultiConfig(self, file):
171			"""
172			returns a dictionary with keys of the form
173			<section>.<option> and the corresponding values
174			"""
175			config={}
176			cp = ConfigParser.ConfigParser()
177			cp.read(file)
178			for sec in cp.sections():
179			# print 'Section',sec
180			config[sec]={}
181			for opt in cp.options(sec):
182			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
183			config[sec][opt] = string.strip(cp.get(sec,opt))
184			return config
185
186			def run(self):
187			#run crabs
188			for sec in self.cfg_params_dataset:
189			options={}
190			if self.flag_continue:
191			options['-c']=sec
192			# DatasetName to be used
193			options['-USER.ui_working_dir']=sec
194			# options from multicrab.cfg
195			for opt in self.cfg_params_dataset[sec]:
196			tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
197			options[tmp]=self.cfg_params_dataset[sec][opt]
198	slacapra	1.2	# add section to storage_path if exist in crab.cfg
199	slacapra	1.5	if self.user_remote_dir:
200			options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
201	slacapra	1.1	# Input options (command)
202			for opt in self.opts:
203			options[opt]=self.opts[opt]
204			try:
205	slacapra	1.3	#print options
206	slacapra	1.1	crab = Crab(options)
207			crab.run()
208			common.apmon.free()
209	slacapra	1.5	del crab
210	slacapra	1.1	except CrabException, e:
211			print '\n' + common.prog_name + ': ' + str(e) + '\n'
212			if common.logger:
213			common.logger.write('ERROR: '+str(e)+'\n')
214			pass
215			pass
216			pass
217			pass
218
219			#common.apmon.free()
220
221			###########################################################################
222			if __name__ == '__main__':
223			## Get rid of some useless warning
224			try:
225			import warnings
226			warnings.simplefilter("ignore", RuntimeWarning)
227			except ImportError:
228			pass # too bad, you'll get the warning
229
230			# Parse command-line options and create a dictionary with
231			# key-value pairs.
232			options = parseOptions(sys.argv[1:])
233
234			# Process "help" options, such as '-help', '-version'
235			if processHelpOptions(options) : sys.exit(0)
236
237			# Create, initialize, and run a Crab object
238			try:
239			multicrab = MultiCrab(options)
240			multicrab.run()
241			except CrabException, e:
242			print '\n' + common.prog_name + ': ' + str(e) + '\n'
243
244			pass