ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.8.2.2 by fanzago, Tue Jun 24 15:20:56 2008 UTC vs.
Revision 1.22 by spiga, Sat Mar 7 16:40:03 2009 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13	<
14	<	#  Naming convention:
14	<	#  methods starting with 'ws' are responsible to provide
15	<	#  corresponding part of the job script ('ws' stands for 'write script').
13	>	import socket
14	>	import sha
15
16		class SchedulerCondor(SchedulerLocal) :
17	+	"""
18	+	Class to implement the vanilla (local) Condor scheduler
19	+	Naming convention:  Methods starting with 'ws' provide
20	+	the corresponding part of the job script
21	+	('ws' stands for 'write script').
22	+	"""
23
24	<	def __init__(self):
25	<	Scheduler.__init__(self,"CONDOR")
26	<	return
27	<
28	<
29	<	def configure(self, cfg_params):
25	<	SchedulerLocal.configure(self, cfg_params)
26	<	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	<
28	<	try:
29	<	tmp =  cfg_params['CMSSW.datasetpath']
30	<	if string.lower(tmp)=='none':
31	<	self.datasetPath = None
32	<	self.selectNoInput = 1
33	<	else:
34	<	self.datasetPath = tmp
35	<	self.selectNoInput = 0
36	<	except KeyError:
37	<	msg = "Error: datasetpath not defined "
38	<	raise CrabException(msg)
24	>	def __init__(self):
25	>	SchedulerLocal.__init__(self,"CONDOR")
26	>	self.datasetPath   = None
27	>	self.selectNoInput = None
28	>	self.return_data   = 0
29	>	self.copy_data     = 0
30
31	<	return
31	>	self.environment_unique_identifier = None
32	>	return
33
34
35	<	def sched_parameter(self,i,task):
36	<	"""
37	<	Return scheduler-specific parameters
38	<	"""
47	<	index = int(common._db.nJobs()) - 1
48	<	sched_param= ''
35	>	def configure(self, cfg_params):
36	>	"""
37	>	Configure the scheduler with the config settings from the user
38	>	"""
39
40	<	for i in range(index):
51	<	pass
40	>	SchedulerLocal.configure(self, cfg_params)
41
42	<	return sched_param
42	>	try:
43	>	tmp =  cfg_params['CMSSW.datasetpath']
44	>	if tmp.lower() == 'none':
45	>	self.datasetPath = None
46	>	self.selectNoInput = 1
47	>	else:
48	>	self.datasetPath = tmp
49	>	self.selectNoInput = 0
50	>	except KeyError:
51	>	msg = "Error: datasetpath not defined "
52	>	raise CrabException(msg)
53
54	+	self.return_data = cfg_params.get('USER.return_data', 0)
55	+	self.copy_data   = cfg_params.get("USER.copy_data", 0)
56
57	<	def realSchedParams(self,cfg_params):
57	<	"""
58	<	Return dictionary with specific parameters, to use
59	<	with real scheduler
60	<	"""
57	>	if int(self.copy_data) == 1:
58
59	<	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
60	<	params = {'tmpDir':tmpDir}
61	<	return  params
59	>	self.proxyValid = 0
60	>	self.dontCheckProxy = int(cfg_params.get("EDG.dont_check_proxy", 0))
61	>	self.proxyServer = cfg_params.get("EDG.proxy_server", 'myproxy.cern.ch')
62	>	common.logger.debug(5,'Setting myproxy server to ' + self.proxyServer)
63
64	+	self.group = cfg_params.get("EDG.group", None)
65	+	self.role  = cfg_params.get("EDG.role", None)
66	+	self.VO    = cfg_params.get('EDG.virtual_organization', 'cms')
67
68	<	def listMatch(self, seList, full, onlyOSG=True):
68	<	"""
69	<	Check the compatibility of available resources
70	<	"""
68	>	self.checkProxy()
69
70	<	# May have problems with onlyOSG being false, probably due to lengths of lists and command line.
73	<	# Either re-write osg_bdii.py with a proper ldap library or break the queries apart
70	>	self.role  = None
71
72	<	if self.selectNoInput:
76	<	return [True]
77	<	else:
78	<	return SchedulerLocal.listMatch(self, seList, full)
72	>	return
73
74	<	def decodeLogInfo(self, file):
75	<	"""
76	<	Parse logging info file and return main info
77	<	"""
84	<	import CondorGLoggingInfo
85	<	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
86	<	reason = loggingInfo.decodeReason(file)
87	<	return reason
74	>	def Env_uniqueId(self):
75	>	taskHash = sha.new(common._db.queryTask('name')).hexdigest()
76	>	id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
77	>	return id
78
79	+	def sched_parameter(self, i, task):
80	+	"""
81	+	Return scheduler-specific parameters
82	+	"""
83
84	<	def wsExitFunc(self):
85	<	"""
92	<	"""
93	<	txt = '\n'
94	<	txt += '#\n'
95	<	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
96	<	txt += '#\n\n'
84	>	index = int(common._db.nJobs()) - 1
85	>	schedParam = ''
86
87	<	txt += 'func_exit() { \n'
88	<	txt += self.wsExitFunc_common()
87	>	for i in range(index):
88	>	pass
89
90	<	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
102	<	txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
103	<	txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'
90	>	return schedParam
91
105	–	txt += '    exit $job_exit_code\n'
106	–	txt += '}\n'
92
93	<	return txt
93	>	def realSchedParams(self, cfg_params):
94	>	"""
95	>	Return dictionary with specific parameters, to use with real scheduler
96	>	"""
97
98	<	def wsInitialEnvironment(self):
99	<	"""
100	<	Returns part of a job script which does scheduler-specific work.
101	<	"""
98	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
99	>	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
100	>	jobDir = common.work_space.jobDir()
101	>	params = {'tmpDir':tmpDir,
102	>	'jobDir':jobDir}
103	>	return params
104	>
105	>
106	>	def listMatch(self, seList, full):
107	>	"""
108	>	Check the compatibility of available resources
109	>	"""
110	>
111	>	return [True]
112	>
113	>
114	>	def decodeLogInfo(self, fileName):
115	>	"""
116	>	Parse logging info file and return main info
117	>	"""
118	>
119	>	import CondorGLoggingInfo
120	>	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
121	>	reason = loggingInfo.decodeReason(fileName)
122	>	return reason
123	>
124	>
125	>	def wsCopyOutput(self):
126	>	"""
127	>	Write a CopyResults part of a job script, e.g.
128	>	to copy produced output into a storage element.
129	>	"""
130	>	txt = self.wsCopyOutput_comm()
131	>	return txt
132	>
133	>
134	>	def wsExitFunc(self):
135	>	"""
136	>	Returns the part of the job script which runs prior to exit
137	>	"""
138	>
139	>	txt = '\n'
140	>	txt += '#\n'
141	>	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
142	>	txt += '#\n\n'
143	>
144	>	txt += 'func_exit() { \n'
145	>	txt += self.wsExitFunc_common()
146	>
147	>	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
148	>	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
149	>	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
150	>
151	>	txt += '    exit $job_exit_code\n'
152	>	txt += '}\n'
153	>
154	>	return txt
155	>
156	>	def wsInitialEnvironment(self):
157	>	"""
158	>	Returns part of a job script which does scheduler-specific work.
159	>	"""
160	>
161	>	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
162	>	txt += 'echo "Beginning environment"\n'
163	>	txt += 'printenv | sort\n'
164
165	<	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
166	<	txt += 'echo "Beginning environment"\n'
167	<	txt += 'printenv | sort\n'
165	>	txt += 'middleware='+self.name()+' \n'
166	>	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
167	>	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
168	>	txt += 'fi\n'
169
170	<	txt += 'middleware='+self.name()+' \n'
120	<	txt += """
170	>	txt += """
171		if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
172	<	ORIG_WD=`pwd`
123	<	echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
172	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
173		if [ -e ../default.tgz ] ;then
174		echo "Found ISB in parent directory (Local Condor)"
175		cp ../default.tgz $_CONDOR_SCRATCH_DIR
#	Line 129 | Line 178 | if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CO
178		fi
179		"""
180
181	<	return txt
181	>	return txt

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines