CRAB/python/multicrab.py

#!/usr/bin/env python
import sys, os, time, string, shutil
from crab_util import *
from crab import *
import common

###########################################################################
class MultiCrab:
    def __init__(self, opts):
        self.prog_name='multicrab'
        # Configuration file
        self.cfg_fname = None
        # Continuation flag
        self.flag_continue = 0
        self.continue_dir = None
        self.processContinueOption_(opts)

        self.processIniFile_(opts)
        # configuration
        self.opts=opts

        if not self.flag_continue:
            self.createWorkSpace()

        print self.prog_name + ' running on ' +  time.ctime(time.time())
        print '  working directory   ' + self.continue_dir


    def processContinueOption_(self,opts):

        # Look for the '-continue' option.

        for opt in opts.keys():
            if ( opt in ('-continue','-c') ):
                self.flag_continue = 1
                val = opts[opt]
                if val:
                    if val[0] == '/': self.continue_dir = val     # abs path
                    else: self.continue_dir = os.getcwd() + '/' + val      # rel path
                    pass
                break
            pass

        # Look for actions which has sense only with '-continue'

        if "-create" not in opts.keys() :
            self.flag_continue = 1

        if not self.flag_continue: return

        if not self.continue_dir:
            prefix = self.prog_name + '_'
            self.continue_dir = findLastWorkDir(prefix)
            pass

        if not self.continue_dir:
            raise CrabException('Cannot find last working directory.')

        if not os.path.exists(self.continue_dir):
            msg = 'Cannot continue because the working directory <'
            msg += self.continue_dir
            msg += '> does not exist.'
            raise CrabException(msg)

        return

    def createWorkSpace(self):
        # create WorkingDir for Multicrab
        if 'MULTICRAB.working_dir' in self.opts.keys():    
            self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
        else:
            current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
            self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time

        if self.continue_dir and not os.path.exists(self.continue_dir):
            try:
                os.mkdir(self.continue_dir)
            except OSError:
                msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        else:
            msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
            raise CrabException(msg)

        shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
        
        return
        
    def processIniFile_(self, opts):
        """
        Processes a configuration INI-file.
        """

        # Extract cfg-file name from the cmd-line options.

        for opt in opts.keys():
            if ( opt == '-cfg' ):
                if self.flag_continue:
                    raise CrabException('-continue and -cfg cannot coexist.')
                if opts[opt] :
                    self.cfg_fname = opts[opt]
                    del opts[opt] # do not pass cfg further on
                else : processHelpOptions()
                pass
            pass

        # Set default cfg-fname

        if self.cfg_fname == None:
            if self.flag_continue:
                self.cfg_fname = self.continue_dir + '/multicrab.cfg'
            else:
                self.cfg_fname = 'multicrab.cfg'
                pass
            pass

        # Load cfg-file

        cfg_params = {}
        if self.cfg_fname != None:
            if os.path.exists(self.cfg_fname):
                cfg_params = self.loadMultiConfig(self.cfg_fname)
                pass
            else:
                msg = 'cfg-file '+self.cfg_fname+' not found.'
                raise CrabException(msg)
                pass
            pass

        # process the [CRAB] section

        lhp = len('MULTICRAB.')
        for k in cfg_params.keys():
            if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
                opt = '-'+k[lhp:]
                if len(opt) >= 3 and opt[:3] == '-__': continue
                if opt not in opts.keys():
                    opts[opt] = cfg_params[k]
                    pass
                pass
            pass

        self.cfg_params_dataset = {}
        common_opts = {}
        # first get common sections
        for sec in cfg_params:
            if sec in ['MULTICRAB']:
                if 'cfg' in cfg_params[sec]:
                    common_opts['cfg']=cfg_params[sec]['cfg']
                continue
            if sec in ['COMMON']:
                common_opts.update(cfg_params[sec])
                continue
            pass

        crab_cfg='crab.cfg'
        if common_opts.has_key('cfg') : crab_cfg=common_opts['cfg']

        # then Dataset's specific
        for sec in cfg_params:
            if sec in ['MULTICRAB', 'COMMON']: continue
            self.cfg_params_dataset[sec]=cfg_params[sec]
            # add common to all dataset
            for key in common_opts:
                self.cfg_params_dataset[sec][key]=common_opts[key]
            pass

        # read crab.cfg file and search for storage_path
        crab_cfg_params = loadConfig(crab_cfg,{})
        if cfg_params.has_key("COMMON"):
            self.user_remote_dir = cfg_params["COMMON"].get("user.user_remote_dir", crab_cfg_params.get("USER.user_remote_dir",None))
        else:
            self.user_remote_dir = crab_cfg_params.get("USER.user_remote_dir",None)
        return

    def loadMultiConfig(self, file):
        """
        returns a dictionary with keys of the form
        <section>.<option> and the corresponding values
        """
        config={}
        cp = ConfigParser.ConfigParser()
        cp.read(file)
        for sec in cp.sections():
            # print 'Section',sec
            config[sec]={}
            for opt in cp.options(sec):
                #print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
                config[sec][opt] = string.strip(cp.get(sec,opt))
        return config

    def run(self):
        #run crabs
        for sec in self.cfg_params_dataset:
            options={}
            if self.flag_continue:
                options['-c']=sec
            # DatasetName to be used
            options['-USER.ui_working_dir']=sec
            # options from multicrab.cfg
            for opt in self.cfg_params_dataset[sec]:
                tmp = "-"+str(opt)
                if len(opt.split("."))==2:
                    tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
                
                options[tmp]=self.cfg_params_dataset[sec][opt]
            # add section to storage_path if exist in crab.cfg
            if not self.cfg_params_dataset.has_key("USER.user_remote_dir") and self.user_remote_dir:
                options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
            # Input options (command)
            for opt in self.opts:
                options[opt]=self.opts[opt]
                if self.flag_continue and options.has_key("-cfg"):
                    del options['-cfg']
                pass
            crab = Crab()
            try:
                crab.initialize_(options)
                crab.run()
                del crab
                print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)  
                print '\n##############################  E N D  ####################################\n'
            except CrabException, e:
                del crab
                print '\n' + common.prog_name + ': ' + str(e) + '\n'
                pass
            pass
            if (common.logger): common.logger.delete()
        pass
        

###########################################################################
if __name__ == '__main__':
    ## Get rid of some useless warning
    try:
        import warnings
        warnings.simplefilter("ignore", RuntimeWarning)
    except ImportError:
        pass # too bad, you'll get the warning

    # Parse command-line options and create a dictionary with
    # key-value pairs.
    options = parseOptions(sys.argv[1:])

    # Process "help" options, such as '-help', '-version'
    if processHelpOptions(options) : sys.exit(0)

    # Create, initialize, and run a Crab object
    try:
        multicrab = MultiCrab(options)
        multicrab.run()
    except CrabException, e:
        print '\n' + common.prog_name + ': ' + str(e) + '\n'

    pass
Revision:	1.15
Committed:	Tue Jun 9 13:13:36 2009 UTC (15 years, 10 months ago) by slacapra
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_6_6_pre6, CRAB_2_6_6_pre5, CRAB_2_6_6_pre4, CRAB_2_6_6_pre3, CRAB_2_6_6_pre2, CRAB_2_6_6_check, CRAB_2_6_6, CRAB_2_6_6_pre1, CRAB_2_6_5, CRAB_2_7_0_pre8, CRAB_2_6_5_pre1, CRAB_2_7_0_pre7, CRAB_2_6_4, CRAB_2_7_0_pre6, CRAB_2_6_4_pre1, CRAB_2_7_0_pre5, CRAB_2_6_3_patch_2, CRAB_2_6_3_patch_2_pre2, CRAB_2_6_3_patch_2_pre1, CRAB_2_6_3_patch_1, CRAB_2_7_0_pre4, CRAB_2_7_0_pre3, CRAB_2_6_3, CRAB_2_6_3_pre5, CRAB_2_6_3_pre4, CRAB_2_6_3_pre3, CRAB_2_6_3_pre2, CRAB_2_7_0_pre2, CRAB_2_6_3_pre1, test_1, CRAB_2_7_0_pre1, CRAB_2_6_2, CRAB_2_6_2_pre2, CRAB_2_6_2_pre1, CRAB_2_6_1_pre4, CRAB_2_6_1_pre3, CRAB_2_6_1_pre2, CRAB_2_6_1_pre1, CRAB_2_6_1, CRAB_2_6_0, CRAB_2_6_0_pre14, CRAB_2_6_0_pre13, CRAB_2_6_0_pre12, CRAB_2_6_0_pre11, CRAB_2_6_0_pre10, CRAB_2_6_0_pre9, CRAB_2_6_0_pre8
Branch point for:	Lumi2_8, CRAB_2_6_X_br
Changes since 1.14:	+5 -5 lines
Log Message:	handle properly, well, with some dirty trick, the cretion and destruction of Crab and CrabLogger objects
#	User	Rev	Content
1	slacapra	1.1	#!/usr/bin/env python
2			import sys, os, time, string, shutil
3			from crab_util import *
4			from crab import *
5			import common
6
7			###########################################################################
8			class MultiCrab:
9			def __init__(self, opts):
10			self.prog_name='multicrab'
11			# Configuration file
12			self.cfg_fname = None
13			# Continuation flag
14			self.flag_continue = 0
15			self.continue_dir = None
16			self.processContinueOption_(opts)
17
18			self.processIniFile_(opts)
19			# configuration
20			self.opts=opts
21
22			if not self.flag_continue:
23			self.createWorkSpace()
24
25			print self.prog_name + ' running on ' + time.ctime(time.time())
26			print ' working directory ' + self.continue_dir
27
28
29			def processContinueOption_(self,opts):
30
31			# Look for the '-continue' option.
32
33			for opt in opts.keys():
34			if ( opt in ('-continue','-c') ):
35			self.flag_continue = 1
36			val = opts[opt]
37			if val:
38			if val[0] == '/': self.continue_dir = val # abs path
39			else: self.continue_dir = os.getcwd() + '/' + val # rel path
40			pass
41			break
42			pass
43
44			# Look for actions which has sense only with '-continue'
45
46			if "-create" not in opts.keys() :
47			self.flag_continue = 1
48
49			if not self.flag_continue: return
50
51			if not self.continue_dir:
52			prefix = self.prog_name + '_'
53			self.continue_dir = findLastWorkDir(prefix)
54			pass
55
56			if not self.continue_dir:
57			raise CrabException('Cannot find last working directory.')
58
59			if not os.path.exists(self.continue_dir):
60			msg = 'Cannot continue because the working directory <'
61			msg += self.continue_dir
62			msg += '> does not exist.'
63			raise CrabException(msg)
64
65			return
66
67			def createWorkSpace(self):
68			# create WorkingDir for Multicrab
69			if 'MULTICRAB.working_dir' in self.opts.keys():
70			self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
71			else:
72			current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
73			self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time
74
75			if self.continue_dir and not os.path.exists(self.continue_dir):
76			try:
77			os.mkdir(self.continue_dir)
78			except OSError:
79			msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
80			raise CrabException(msg)
81			pass
82			else:
83			msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
84			raise CrabException(msg)
85
86			shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
87
88			return
89
90			def processIniFile_(self, opts):
91			"""
92			Processes a configuration INI-file.
93			"""
94
95			# Extract cfg-file name from the cmd-line options.
96
97			for opt in opts.keys():
98			if ( opt == '-cfg' ):
99			if self.flag_continue:
100			raise CrabException('-continue and -cfg cannot coexist.')
101	slacapra	1.8	if opts[opt] :
102			self.cfg_fname = opts[opt]
103	slacapra	1.11	del opts[opt] # do not pass cfg further on
104	slacapra	1.1	else : processHelpOptions()
105			pass
106			pass
107
108			# Set default cfg-fname
109
110			if self.cfg_fname == None:
111			if self.flag_continue:
112			self.cfg_fname = self.continue_dir + '/multicrab.cfg'
113			else:
114			self.cfg_fname = 'multicrab.cfg'
115			pass
116			pass
117
118			# Load cfg-file
119
120	slacapra	1.7	cfg_params = {}
121	slacapra	1.1	if self.cfg_fname != None:
122			if os.path.exists(self.cfg_fname):
123	slacapra	1.7	cfg_params = self.loadMultiConfig(self.cfg_fname)
124	slacapra	1.1	pass
125			else:
126			msg = 'cfg-file '+self.cfg_fname+' not found.'
127			raise CrabException(msg)
128			pass
129			pass
130
131			# process the [CRAB] section
132
133			lhp = len('MULTICRAB.')
134	slacapra	1.7	for k in cfg_params.keys():
135	slacapra	1.1	if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
136			opt = '-'+k[lhp:]
137			if len(opt) >= 3 and opt[:3] == '-__': continue
138			if opt not in opts.keys():
139	slacapra	1.7	opts[opt] = cfg_params[k]
140	slacapra	1.1	pass
141			pass
142			pass
143
144			self.cfg_params_dataset = {}
145	slacapra	1.3	common_opts = {}
146	slacapra	1.1	# first get common sections
147	slacapra	1.7	for sec in cfg_params:
148	slacapra	1.1	if sec in ['MULTICRAB']:
149	slacapra	1.8	if 'cfg' in cfg_params[sec]:
150			common_opts['cfg']=cfg_params[sec]['cfg']
151	slacapra	1.1	continue
152			if sec in ['COMMON']:
153	slacapra	1.8	common_opts.update(cfg_params[sec])
154	slacapra	1.1	continue
155			pass
156
157	slacapra	1.8	crab_cfg='crab.cfg'
158			if common_opts.has_key('cfg') : crab_cfg=common_opts['cfg']
159
160	slacapra	1.1	# then Dataset's specific
161	slacapra	1.7	for sec in cfg_params:
162	slacapra	1.1	if sec in ['MULTICRAB', 'COMMON']: continue
163	slacapra	1.7	self.cfg_params_dataset[sec]=cfg_params[sec]
164	slacapra	1.1	# add common to all dataset
165	slacapra	1.3	for key in common_opts:
166			self.cfg_params_dataset[sec][key]=common_opts[key]
167	slacapra	1.1	pass
168
169	slacapra	1.2	# read crab.cfg file and search for storage_path
170	slacapra	1.8	crab_cfg_params = loadConfig(crab_cfg,{})
171	slacapra	1.7	if cfg_params.has_key("COMMON"):
172			self.user_remote_dir = cfg_params["COMMON"].get("user.user_remote_dir", crab_cfg_params.get("USER.user_remote_dir",None))
173	slacapra	1.6	else:
174	slacapra	1.7	self.user_remote_dir = crab_cfg_params.get("USER.user_remote_dir",None)
175	slacapra	1.1	return
176
177			def loadMultiConfig(self, file):
178			"""
179			returns a dictionary with keys of the form
180			<section>.<option> and the corresponding values
181			"""
182			config={}
183			cp = ConfigParser.ConfigParser()
184			cp.read(file)
185			for sec in cp.sections():
186			# print 'Section',sec
187			config[sec]={}
188			for opt in cp.options(sec):
189			#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
190			config[sec][opt] = string.strip(cp.get(sec,opt))
191			return config
192
193			def run(self):
194			#run crabs
195			for sec in self.cfg_params_dataset:
196			options={}
197			if self.flag_continue:
198			options['-c']=sec
199			# DatasetName to be used
200			options['-USER.ui_working_dir']=sec
201			# options from multicrab.cfg
202			for opt in self.cfg_params_dataset[sec]:
203	slacapra	1.8	tmp = "-"+str(opt)
204			if len(opt.split("."))==2:
205			tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
206
207	slacapra	1.1	options[tmp]=self.cfg_params_dataset[sec][opt]
208	slacapra	1.2	# add section to storage_path if exist in crab.cfg
209	slacapra	1.6	if not self.cfg_params_dataset.has_key("USER.user_remote_dir") and self.user_remote_dir:
210	slacapra	1.5	options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
211	slacapra	1.1	# Input options (command)
212			for opt in self.opts:
213			options[opt]=self.opts[opt]
214	slacapra	1.9	if self.flag_continue and options.has_key("-cfg"):
215			del options['-cfg']
216	slacapra	1.13	pass
217	slacapra	1.15	crab = Crab()
218	slacapra	1.1	try:
219	slacapra	1.15	crab.initialize_(options)
220	slacapra	1.1	crab.run()
221	slacapra	1.15	del crab
222	slacapra	1.13	print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)
223	slacapra	1.10	print '\n############################## E N D ####################################\n'
224	slacapra	1.1	except CrabException, e:
225	slacapra	1.15	del crab
226	slacapra	1.1	print '\n' + common.prog_name + ': ' + str(e) + '\n'
227			pass
228			pass
229	slacapra	1.15	if (common.logger): common.logger.delete()
230	slacapra	1.1	pass
231
232
233			###########################################################################
234			if __name__ == '__main__':
235			## Get rid of some useless warning
236			try:
237			import warnings
238			warnings.simplefilter("ignore", RuntimeWarning)
239			except ImportError:
240			pass # too bad, you'll get the warning
241
242			# Parse command-line options and create a dictionary with
243			# key-value pairs.
244			options = parseOptions(sys.argv[1:])
245
246			# Process "help" options, such as '-help', '-version'
247			if processHelpOptions(options) : sys.exit(0)
248
249			# Create, initialize, and run a Crab object
250			try:
251			multicrab = MultiCrab(options)
252			multicrab.run()
253			except CrabException, e:
254			print '\n' + common.prog_name + ': ' + str(e) + '\n'
255
256			pass