ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/multicrab.py

Comparing COMP/CRAB/python/multicrab.py (file contents):
Revision 1.2 by slacapra, Mon Jul 21 11:11:07 2008 UTC vs.
Revision 1.32 by slacapra, Thu Apr 26 14:10:25 2012 UTC

#	Line 25 | Line 25 | class MultiCrab:
25		print self.prog_name + ' running on ' +  time.ctime(time.time())
26		print '  working directory   ' + self.continue_dir
27
28	–	crabs=[]
28
29		def processContinueOption_(self,opts):
30
#	Line 67 | Line 66 | class MultiCrab:
66
67		def createWorkSpace(self):
68		# create WorkingDir for Multicrab
69	<	if 'MULTICRAB.working_dir' in self.opts.keys():
70	<	self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
69	>	import os
70	>	if not self.continue_dir:
71	>	prefix = self.prog_name + '_'
72	>	self.continue_dir = findLastWorkDir(prefix)
73	>	pass
74	>	# if 'MULTICRAB.working_dir' in self.opts.keys():
75	>	#     self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
76	>	if self.ui_working_dir:
77	>	self.continue_dir = os.path.abspath(self.ui_working_dir)
78		else:
79		current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
80		self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time
#	Line 77 | Line 83 | class MultiCrab:
83		try:
84		os.mkdir(self.continue_dir)
85		except OSError:
86	<	msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
86	>	msg = 'ERROR: Cannot create '+str(self.continue_dir) +' directory.\n'
87		raise CrabException(msg)
88		pass
89		else:
90	<	msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
90	>	msg = 'ERROR: Directory '+str(self.continue_dir) +' already exist.\n'
91		raise CrabException(msg)
92
93	<	shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
93	>	os.putenv("MULTICRAB_WORKDIR",self.continue_dir)
94	>	shutil.copyfile(self.cfg_fname,self.continue_dir+'/multicrab.cfg')
95
96		return
97
#	Line 99 | Line 106 | class MultiCrab:
106		if ( opt == '-cfg' ):
107		if self.flag_continue:
108		raise CrabException('-continue and -cfg cannot coexist.')
109	<	if opts[opt] : self.cfg_fname = opts[opt]
109	>	if opts[opt] :
110	>	self.cfg_fname = opts[opt]
111	>	del opts[opt] # do not pass cfg further on
112		else : processHelpOptions()
113		pass
114		pass
#	Line 116 | Line 125 | class MultiCrab:
125
126		# Load cfg-file
127
128	+	cfg_params = {}
129		if self.cfg_fname != None:
130		if os.path.exists(self.cfg_fname):
131	<	self.cfg_params = self.loadMultiConfig(self.cfg_fname)
131	>	cfg_params = self.loadMultiConfig(self.cfg_fname)
132		pass
133		else:
134	<	msg = 'cfg-file '+self.cfg_fname+' not found.'
134	>	msg = 'ERROR: cfg-file '+self.cfg_fname+' not found.'
135		raise CrabException(msg)
136		pass
137		pass
#	Line 129 | Line 139 | class MultiCrab:
139		# process the [CRAB] section
140
141		lhp = len('MULTICRAB.')
142	<	for k in self.cfg_params.keys():
142	>	for k in cfg_params.keys():
143		if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
144		opt = '-'+k[lhp:]
145		if len(opt) >= 3 and opt[:3] == '-__': continue
146		if opt not in opts.keys():
147	<	opts[opt] = self.cfg_params[k]
147	>	opts[opt] = cfg_params[k]
148		pass
149		pass
150		pass
151
152		self.cfg_params_dataset = {}
153	<	common_opts = []
153	>	common_opts = {}
154		# first get common sections
155	<	for sec in self.cfg_params:
155	>	for sec in cfg_params:
156		if sec in ['MULTICRAB']:
157	<	cfg_common=self.cfg_params[sec]
157	>	if 'cfg' in cfg_params[sec]:
158	>	common_opts['cfg']=cfg_params[sec]['cfg']
159		continue
160		if sec in ['COMMON']:
161	<	common_opts.append(self.cfg_params[sec])
161	>	common_opts.update(cfg_params[sec])
162		continue
163		pass
164
165	+	crab_cfg='crab.cfg'
166	+	# read crab.cfg file and search for storage_path
167	+	crab_cfg_params = loadConfig(crab_cfg,{})
168	+	# also USER.ui_working_dir USER.outputdir and USER.logdir need special treatment
169	+	if cfg_params.has_key("COMMON"):
170	+	self.user_remote_dir = cfg_params["COMMON"].get("user.user_remote_dir", crab_cfg_params.get("USER.user_remote_dir",None))
171	+	self.outputdir = cfg_params["COMMON"].get("user.outputdir", crab_cfg_params.get("USER.outputdir",None))
172	+	self.logdir = cfg_params["COMMON"].get("user.logdir", crab_cfg_params.get("USER.logdir",None))
173	+	self.ui_working_dir = cfg_params["COMMON"].get("user.ui_working_dir", crab_cfg_params.get("USER.ui_working_dir",None))
174	+	self.publish_data_name = cfg_params["COMMON"].get("user.publish_data_name", crab_cfg_params.get("USER.publish_data_name",None))
175	+	else:
176	+	self.user_remote_dir = crab_cfg_params.get("USER.user_remote_dir","./")
177	+	self.outputdir = crab_cfg_params.get("USER.outputdir",None)
178	+	self.logdir = crab_cfg_params.get("USER.logdir",None)
179	+	self.ui_working_dir = crab_cfg_params.get("USER.ui_working_dir",None)
180	+	self.publish_data_name = crab_cfg_params.get("USER.publish_data_name",None)
181	+	print "self.publish_data_name ",self.publish_data_name
182	+
183	+	if common_opts.has_key('cfg') : crab_cfg=common_opts['cfg']
184	+
185		# then Dataset's specific
186	<	for sec in self.cfg_params:
186	>	for sec in cfg_params:
187		if sec in ['MULTICRAB', 'COMMON']: continue
157	–	self.cfg_params_dataset[sec]=self.cfg_params[sec]
188		# add common to all dataset
189	<	for opt in common_opts:
190	<	self.cfg_params_dataset[sec]=opt
189	>	self.cfg_params_dataset[sec]=cfg_params[sec]
190	>	# special tratment for some parameter
191	>	if not self.cfg_params_dataset[sec].has_key("user.publish_data_name") and self.publish_data_name:
192	>	self.cfg_params_dataset[sec]["user.publish_data_name"]=self.publish_data_name+"_"+sec
193	>	if not self.cfg_params_dataset[sec].has_key("user.user_remote_dir") and self.user_remote_dir:
194	>	self.cfg_params_dataset[sec]["user.user_remote_dir"]=self.user_remote_dir+"/"+sec
195	>	if not self.cfg_params_dataset[sec].has_key("user.ui_working_dir") and self.ui_working_dir:
196	>	self.cfg_params_dataset[sec]["user.ui_working_dir"]=self.ui_working_dir+"/"+sec
197	>	if not self.cfg_params_dataset[sec].has_key("user.logdir") and self.logdir:
198	>	self.cfg_params_dataset[sec]["user.logdir"]=self.logdir+"/"+sec
199	>	if not self.cfg_params_dataset[sec].has_key("user.outputdir") and self.outputdir:
200	>	self.cfg_params_dataset[sec]["user.outputdir"]=self.outputdir+"/"+sec
201	>	for key in common_opts:
202	>	if not self.cfg_params_dataset[sec].has_key(key):
203	>	self.cfg_params_dataset[sec][key]=common_opts[key]
204		pass
205
163	–	self.cfg=cfg_common['cfg']
206
165	–	# read crab.cfg file and search for storage_path
166	–	cfg_params = loadConfig(self.cfg,{})
167	–	self.storage_path = cfg_params.get("USER.storage_path",None)
207		return
208
209		def loadMultiConfig(self, file):
#	Line 179 | Line 218 | class MultiCrab:
218		# print 'Section',sec
219		config[sec]={}
220		for opt in cp.options(sec):
221	<	#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
221	>	# print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
222		config[sec][opt] = string.strip(cp.get(sec,opt))
223		return config
224
225		def run(self):
226		#run crabs
227	+	runFileName = self.continue_dir+'/multicrab.exe'
228	+	runFile = open(runFileName,"w")
229		for sec in self.cfg_params_dataset:
230		options={}
231		if self.flag_continue:
#	Line 193 | Line 234 | class MultiCrab:
234		options['-USER.ui_working_dir']=sec
235		# options from multicrab.cfg
236		for opt in self.cfg_params_dataset[sec]:
237	<	tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
237	>	tmp = "-"+str(opt)
238	>	if len(opt.split("."))==2:
239	>	tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
240	>
241		options[tmp]=self.cfg_params_dataset[sec][opt]
242	+
243	+	# if ui_working_dir is set, change -c dir accordnigly
244	+	if not self.cfg_params_dataset.has_key("USER.ui_working_dir") and self.ui_working_dir:
245	+	if self.flag_continue:
246	+	options['-c']=self.ui_working_dir+"/"+sec
247	+
248	+	# check if user_remote_dir is set in multicrab.cfg
249	+	# protect against no user_remote_dir
250	+	# self.user_remote_dir =self.cfg_params_dataset[sec].get("user.user_remote_dir",self.user_remote_dir)
251	+	# if not self.user_remote_dir:
252	+	#     self.user_remote_dir = "./"
253		# add section to storage_path if exist in crab.cfg
254	<	if self.storage_path:
255	<	options["-USER.storage_path"]=self.storage_path+"/"+sec
254	>	# if not self.cfg_params_dataset.has_key("USER.user_remote_dir") and self.user_remote_dir:
255	>	#     options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
256	>	# print options["-USER.user_remote_dir"]
257	>	# also for ui_working_dir
258	>	# if not self.cfg_params_dataset.has_key("USER.ui_working_dir") and self.ui_working_dir:
259	>	#     options["-USER.ui_working_dir"]=self.ui_working_dir+"/"+sec
260	>	# also for logDir
261	>	# if not self.cfg_params_dataset.has_key("USER.logdir") and self.logdir:
262	>	#     options["-USER.logdir"]=self.logdir+"/"+sec
263	>	# # also for outputdir
264	>	# if not self.cfg_params_dataset.has_key("USER.outputdir") and self.outputdir:
265	>	#     options["-USER.outputdir"]=self.outputdir+"/"+sec
266	>	# also for publish_data_name
267	>	# print sec," ",self.cfg_params_dataset[sec], self.cfg_params_dataset[sec].has_key("user.publish_data_name")
268	>	# if not self.cfg_params_dataset[sec].has_key("user.publish_data_name") and self.publish_data_name:
269	>	#     options["-USER.publish_data_name"]=self.publish_data_name+"_"+sec
270	>	#     print "adding user.publish_data_name", self.cfg_params_dataset.has_key("user.publish_data_name")
271	>
272		# Input options (command)
273		for opt in self.opts:
274	<	options[opt]=self.opts[opt]
275	<	try:
276	<	# print options
277	<	crab = Crab(options)
278	<	crab.run()
208	<	common.apmon.free()
209	<	except CrabException, e:
210	<	print '\n' + common.prog_name + ': ' + str(e) + '\n'
211	<	if common.logger:
212	<	common.logger.write('ERROR: '+str(e)+'\n')
274	>	if opt != '-c':
275	>	options[opt]=self.opts[opt]
276	>	# options[opt]=self.opts[opt]
277	>	if self.flag_continue and options.has_key("-cfg"):
278	>	del options['-cfg']
279		pass
280	+
281	+
282	+	# write crab command to be executed later...
283	+	cmd='crab '
284	+	for o in options:
285	+	if options[o]==None:
286	+	cmd+=str(o)+' '
287	+	else:
288	+	options[o] = ''.join(options[o].split())
289	+	cmd+=str(o)+'='+str(options[o])+' '
290		pass
291	<	pass
291	>	cmd+="\n"
292	>	#print cmd
293	>
294	>	runFile.write(cmd)
295	>
296	>	# SL this does not work for complex, multi include pset.py
297	>
298	>	# crab = Crab()
299	>	# try:
300	>	#     crab.initialize_(options)
301	>	#     crab.run()
302	>	#     del crab
303	>	#     print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)
304	>	#     print '\n##############################  E N D  ####################################\n'
305	>	# except CrabException, e:
306	>	#     del crab
307	>	#     print '\n' + common.prog_name + ': ' + str(e) + '\n'
308	>	#     pass
309	>	# pass
310	>	# if (common.logger): common.logger.delete()
311		pass
312	+	return self.continue_dir
313
218	–	#common.apmon.free()
314
315		###########################################################################
316		if __name__ == '__main__':
#	Line 236 | Line 331 | if __name__ == '__main__':
331		# Create, initialize, and run a Crab object
332		try:
333		multicrab = MultiCrab(options)
334	<	multicrab.run()
334	>	continue_dir = multicrab.run()
335	>	import os
336	>	sys.exit(continue_dir)
337		except CrabException, e:
338		print '\n' + common.prog_name + ': ' + str(e) + '\n'
339

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines