ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/multicrab.py

Comparing COMP/CRAB/python/multicrab.py (file contents):
Revision 1.4 by slacapra, Mon Nov 10 16:42:22 2008 UTC vs.
Revision 1.33 by slacapra, Thu Sep 20 13:09:26 2012 UTC

#	Line 25 | Line 25 | class MultiCrab:
25		print self.prog_name + ' running on ' +  time.ctime(time.time())
26		print '  working directory   ' + self.continue_dir
27
28	–	crabs=[]
28
29		def processContinueOption_(self,opts):
30
#	Line 67 | Line 66 | class MultiCrab:
66
67		def createWorkSpace(self):
68		# create WorkingDir for Multicrab
69	<	if 'MULTICRAB.working_dir' in self.opts.keys():
70	<	self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
69	>	import os
70	>	if not self.continue_dir:
71	>	prefix = self.prog_name + '_'
72	>	self.continue_dir = findLastWorkDir(prefix)
73	>	pass
74	>	# if 'MULTICRAB.working_dir' in self.opts.keys():
75	>	#     self.continue_dir = os.path.abspath(self.opts['MULTICRAB.working_dir'])
76	>	if self.ui_working_dir:
77	>	self.continue_dir = os.path.abspath(self.ui_working_dir)
78		else:
79		current_time = time.strftime('%y%m%d_%H%M%S', time.localtime(time.time()))
80		self.continue_dir = os.getcwd() + '/' + self.prog_name + '_' + current_time
#	Line 77 | Line 83 | class MultiCrab:
83		try:
84		os.mkdir(self.continue_dir)
85		except OSError:
86	<	msg = 'Cannot create '+str(self.continue_dir) +' directory.\n'
86	>	msg = 'ERROR: Cannot create '+str(self.continue_dir) +' directory.\n'
87		raise CrabException(msg)
88		pass
89		else:
90	<	msg = 'Directory '+str(self.continue_dir) +' already exist.\n'
90	>	msg = 'ERROR: Directory '+str(self.continue_dir) +' already exist.\n'
91		raise CrabException(msg)
92
93	<	shutil.copyfile('multicrab.cfg',self.continue_dir+'/multicrab.cfg')
93	>	os.putenv("MULTICRAB_WORKDIR",self.continue_dir)
94	>	shutil.copyfile(self.cfg_fname,self.continue_dir+'/multicrab.cfg')
95
96		return
97
#	Line 99 | Line 106 | class MultiCrab:
106		if ( opt == '-cfg' ):
107		if self.flag_continue:
108		raise CrabException('-continue and -cfg cannot coexist.')
109	<	if opts[opt] : self.cfg_fname = opts[opt]
109	>	if opts[opt] :
110	>	self.cfg_fname = opts[opt]
111	>	del opts[opt] # do not pass cfg further on
112		else : processHelpOptions()
113		pass
114		pass
#	Line 116 | Line 125 | class MultiCrab:
125
126		# Load cfg-file
127
128	+	cfg_params = {}
129		if self.cfg_fname != None:
130		if os.path.exists(self.cfg_fname):
131	<	self.cfg_params = self.loadMultiConfig(self.cfg_fname)
131	>	cfg_params = self.loadMultiConfig(self.cfg_fname)
132		pass
133		else:
134	<	msg = 'cfg-file '+self.cfg_fname+' not found.'
134	>	msg = 'ERROR: cfg-file '+self.cfg_fname+' not found.'
135		raise CrabException(msg)
136		pass
137		pass
#	Line 129 | Line 139 | class MultiCrab:
139		# process the [CRAB] section
140
141		lhp = len('MULTICRAB.')
142	<	for k in self.cfg_params.keys():
142	>	for k in cfg_params.keys():
143		if len(k) >= lhp and k[:lhp] == 'MULTICRAB.':
144		opt = '-'+k[lhp:]
145		if len(opt) >= 3 and opt[:3] == '-__': continue
146		if opt not in opts.keys():
147	<	opts[opt] = self.cfg_params[k]
147	>	opts[opt] = cfg_params[k]
148		pass
149		pass
150		pass
#	Line 142 | Line 152 | class MultiCrab:
152		self.cfg_params_dataset = {}
153		common_opts = {}
154		# first get common sections
155	<	for sec in self.cfg_params:
155	>	crab_cfg='crab.cfg' # this is the default
156	>	for sec in cfg_params:
157		if sec in ['MULTICRAB']:
158	<	cfg_common=self.cfg_params[sec]
158	>	if 'cfg' in cfg_params[sec]:
159	>	common_opts['cfg']=cfg_params[sec]['cfg']
160	>	crab_cfg=common_opts['cfg'];
161		continue
162		if sec in ['COMMON']:
163	<	common_opts=self.cfg_params[sec]
163	>	common_opts.update(cfg_params[sec])
164		continue
165		pass
166
167	+	# read crab.cfg file and search for storage_path
168	+	crab_cfg_params = loadConfig(crab_cfg,{})
169	+	# also USER.ui_working_dir USER.outputdir and USER.logdir need special treatment
170	+	if cfg_params.has_key("COMMON"):
171	+	self.user_remote_dir = cfg_params["COMMON"].get("user.user_remote_dir", crab_cfg_params.get("USER.user_remote_dir",None))
172	+	self.outputdir = cfg_params["COMMON"].get("user.outputdir", crab_cfg_params.get("USER.outputdir",None))
173	+	self.logdir = cfg_params["COMMON"].get("user.logdir", crab_cfg_params.get("USER.logdir",None))
174	+	self.ui_working_dir = cfg_params["COMMON"].get("user.ui_working_dir", crab_cfg_params.get("USER.ui_working_dir",None))
175	+	self.publish_data_name = cfg_params["COMMON"].get("user.publish_data_name", crab_cfg_params.get("USER.publish_data_name",None))
176	+	else:
177	+	self.user_remote_dir = crab_cfg_params.get("USER.user_remote_dir","./")
178	+	self.outputdir = crab_cfg_params.get("USER.outputdir",None)
179	+	self.logdir = crab_cfg_params.get("USER.logdir",None)
180	+	self.ui_working_dir = crab_cfg_params.get("USER.ui_working_dir",None)
181	+	self.publish_data_name = crab_cfg_params.get("USER.publish_data_name",None)
182	+
183	+	if common_opts.has_key('cfg') : crab_cfg=common_opts['cfg']
184	+
185		# then Dataset's specific
186	<	for sec in self.cfg_params:
186	>	for sec in cfg_params:
187		if sec in ['MULTICRAB', 'COMMON']: continue
157	–	self.cfg_params_dataset[sec]=self.cfg_params[sec]
188		# add common to all dataset
189	+	self.cfg_params_dataset[sec]=cfg_params[sec]
190	+	# special tratment for some parameter
191	+	if not self.cfg_params_dataset[sec].has_key("user.publish_data_name") and self.publish_data_name:
192	+	self.cfg_params_dataset[sec]["user.publish_data_name"]=self.publish_data_name+"_"+sec
193	+	if not self.cfg_params_dataset[sec].has_key("user.user_remote_dir") and self.user_remote_dir:
194	+	self.cfg_params_dataset[sec]["user.user_remote_dir"]=self.user_remote_dir+"/"+sec
195	+	if not self.cfg_params_dataset[sec].has_key("user.ui_working_dir") and self.ui_working_dir:
196	+	self.cfg_params_dataset[sec]["user.ui_working_dir"]=self.ui_working_dir+"/"+sec
197	+	if not self.cfg_params_dataset[sec].has_key("user.logdir") and self.logdir:
198	+	self.cfg_params_dataset[sec]["user.logdir"]=self.logdir+"/"+sec
199	+	if not self.cfg_params_dataset[sec].has_key("user.outputdir") and self.outputdir:
200	+	self.cfg_params_dataset[sec]["user.outputdir"]=self.outputdir+"/"+sec
201		for key in common_opts:
202	<	self.cfg_params_dataset[sec][key]=common_opts[key]
202	>	if not self.cfg_params_dataset[sec].has_key(key):
203	>	self.cfg_params_dataset[sec][key]=common_opts[key]
204		pass
205
163	–	self.cfg=cfg_common['cfg']
164	–
165	–	# read crab.cfg file and search for storage_path
166	–	cfg_params = loadConfig(self.cfg,{})
167	–	self.storage_path = cfg_params.get("USER.storage_path",None)
206		return
207
208		def loadMultiConfig(self, file):
#	Line 179 | Line 217 | class MultiCrab:
217		# print 'Section',sec
218		config[sec]={}
219		for opt in cp.options(sec):
220	<	#print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
220	>	# print 'config['+sec+'.'+opt+'] = '+string.strip(cp.get(sec,opt))
221		config[sec][opt] = string.strip(cp.get(sec,opt))
222		return config
223
224		def run(self):
225		#run crabs
226	+	runFileName = self.continue_dir+'/multicrab.exe'
227	+	runFile = open(runFileName,"w")
228		for sec in self.cfg_params_dataset:
229		options={}
230		if self.flag_continue:
#	Line 193 | Line 233 | class MultiCrab:
233		options['-USER.ui_working_dir']=sec
234		# options from multicrab.cfg
235		for opt in self.cfg_params_dataset[sec]:
236	<	tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
236	>	tmp = "-"+str(opt)
237	>	if len(opt.split("."))==2:
238	>	tmp="-"+string.upper(opt.split(".")[0])+"."+opt.split(".")[1]
239	>
240		options[tmp]=self.cfg_params_dataset[sec][opt]
241	+
242	+	# if ui_working_dir is set, change -c dir accordnigly
243	+	if not self.cfg_params_dataset.has_key("USER.ui_working_dir") and self.ui_working_dir:
244	+	if self.flag_continue:
245	+	options['-c']=self.ui_working_dir+"/"+sec
246	+
247	+	# check if user_remote_dir is set in multicrab.cfg
248	+	# protect against no user_remote_dir
249	+	# self.user_remote_dir =self.cfg_params_dataset[sec].get("user.user_remote_dir",self.user_remote_dir)
250	+	# if not self.user_remote_dir:
251	+	#     self.user_remote_dir = "./"
252		# add section to storage_path if exist in crab.cfg
253	<	if self.storage_path:
254	<	options["-USER.user_remote_dir"]=self.storage_path+"/"+sec
253	>	# if not self.cfg_params_dataset.has_key("USER.user_remote_dir") and self.user_remote_dir:
254	>	#     options["-USER.user_remote_dir"]=self.user_remote_dir+"/"+sec
255	>	# print options["-USER.user_remote_dir"]
256	>	# also for ui_working_dir
257	>	# if not self.cfg_params_dataset.has_key("USER.ui_working_dir") and self.ui_working_dir:
258	>	#     options["-USER.ui_working_dir"]=self.ui_working_dir+"/"+sec
259	>	# also for logDir
260	>	# if not self.cfg_params_dataset.has_key("USER.logdir") and self.logdir:
261	>	#     options["-USER.logdir"]=self.logdir+"/"+sec
262	>	# # also for outputdir
263	>	# if not self.cfg_params_dataset.has_key("USER.outputdir") and self.outputdir:
264	>	#     options["-USER.outputdir"]=self.outputdir+"/"+sec
265	>	# also for publish_data_name
266	>	# print sec," ",self.cfg_params_dataset[sec], self.cfg_params_dataset[sec].has_key("user.publish_data_name")
267	>	# if not self.cfg_params_dataset[sec].has_key("user.publish_data_name") and self.publish_data_name:
268	>	#     options["-USER.publish_data_name"]=self.publish_data_name+"_"+sec
269	>	#     print "adding user.publish_data_name", self.cfg_params_dataset.has_key("user.publish_data_name")
270	>
271		# Input options (command)
272		for opt in self.opts:
273	<	options[opt]=self.opts[opt]
274	<	try:
275	<	#print options
276	<	crab = Crab(options)
277	<	crab.run()
208	<	common.apmon.free()
209	<	except CrabException, e:
210	<	print '\n' + common.prog_name + ': ' + str(e) + '\n'
211	<	if common.logger:
212	<	common.logger.write('ERROR: '+str(e)+'\n')
273	>	if opt != '-c':
274	>	options[opt]=self.opts[opt]
275	>	# options[opt]=self.opts[opt]
276	>	if self.flag_continue and options.has_key("-cfg"):
277	>	del options['-cfg']
278		pass
279	+
280	+
281	+	# write crab command to be executed later...
282	+	cmd='crab '
283	+	for o in options:
284	+	if options[o]==None:
285	+	cmd+=str(o)+' '
286	+	else:
287	+	options[o] = ''.join(options[o].split())
288	+	cmd+=str(o)+'='+str(options[o])+' '
289		pass
290	<	pass
290	>	cmd+="\n"
291	>	#print cmd
292	>
293	>	runFile.write(cmd)
294	>
295	>	# SL this does not work for complex, multi include pset.py
296	>
297	>	# crab = Crab()
298	>	# try:
299	>	#     crab.initialize_(options)
300	>	#     crab.run()
301	>	#     del crab
302	>	#     print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)
303	>	#     print '\n##############################  E N D  ####################################\n'
304	>	# except CrabException, e:
305	>	#     del crab
306	>	#     print '\n' + common.prog_name + ': ' + str(e) + '\n'
307	>	#     pass
308	>	# pass
309	>	# if (common.logger): common.logger.delete()
310		pass
311	+	return self.continue_dir
312
218	–	#common.apmon.free()
313
314		###########################################################################
315		if __name__ == '__main__':
#	Line 236 | Line 330 | if __name__ == '__main__':
330		# Create, initialize, and run a Crab object
331		try:
332		multicrab = MultiCrab(options)
333	<	multicrab.run()
333	>	continue_dir = multicrab.run()
334	>	import os
335	>	sys.exit(continue_dir)
336		except CrabException, e:
337		print '\n' + common.prog_name + ': ' + str(e) + '\n'
338

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines