ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerCondor.py

Comparing COMP/CRAB/python/SchedulerCondor.py (file contents):
Revision 1.8 by spiga, Wed Jun 11 10:03:07 2008 UTC vs.
Revision 1.27 by ewv, Tue Aug 18 15:00:37 2009 UTC

#	Line 1 | Line 1
1	+	"""
2	+	Implements the vanilla (local) Condor scheduler
3	+	"""
4	+
5		__revision__ = "$Id$"
6		__version__ = "$Revision$"
7
8	<	from Scheduler import Scheduler
9	<	from SchedulerLocal import SchedulerLocal
6	<	from crab_exceptions import *
7	<	from crab_util import *
8	<	from crab_logger import Logger
9	<	import common
8	>	from SchedulerLocal  import SchedulerLocal
9	>	from crab_exceptions import CrabException
10
11	+	import common
12		import os
13	<
14	<	#  Naming convention:
14	<	#  methods starting with 'ws' are responsible to provide
15	<	#  corresponding part of the job script ('ws' stands for 'write script').
13	>	import socket
14	>	import sha
15
16		class SchedulerCondor(SchedulerLocal) :
18	–
19	–	def __init__(self):
20	–	Scheduler.__init__(self,"CONDOR")
21	–	return
22	–
23	–
24	–	def configure(self, cfg_params):
25	–	SchedulerLocal.configure(self, cfg_params)
26	–	self.environment_unique_identifier ='${HOSTNAME}_${CONDOR_ID}_' + common._db.queryTask('name')
27	–
28	–	try:
29	–	tmp =  cfg_params['CMSSW.datasetpath']
30	–	if string.lower(tmp)=='none':
31	–	self.datasetPath = None
32	–	self.selectNoInput = 1
33	–	else:
34	–	self.datasetPath = tmp
35	–	self.selectNoInput = 0
36	–	except KeyError:
37	–	msg = "Error: datasetpath not defined "
38	–	raise CrabException(msg)
39	–
40	–	return
41	–
42	–
43	–	def sched_parameter(self,i,task):
44	–	"""
45	–	Return scheduler-specific parameters
46	–	"""
47	–	index = int(common._db.nJobs()) - 1
48	–	sched_param= ''
49	–
50	–	for i in range(index):
51	–	pass
52	–
53	–	return sched_param
54	–
55	–
56	–	def realSchedParams(self,cfg_params):
57	–	"""
58	–	Return dictionary with specific parameters, to use
59	–	with real scheduler
60	–	"""
61	–
62	–	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
63	–	params = {'tmpDir':tmpDir}
64	–	return  params
65	–
66	–
67	–	def listMatch(self, seList, full, onlyOSG=True):
68	–	"""
69	–	Check the compatibility of available resources
17		"""
18	+	Class to implement the vanilla (local) Condor scheduler
19	+	Naming convention:  Methods starting with 'ws' provide
20	+	the corresponding part of the job script
21	+	('ws' stands for 'write script').
22	+	"""
23	+
24	+	def __init__(self):
25	+	SchedulerLocal.__init__(self,"CONDOR")
26	+	self.datasetPath   = None
27	+	self.selectNoInput = None
28	+	self.return_data   = 0
29	+	self.copy_data     = 0
30	+
31	+	self.environment_unique_identifier = None
32	+	return
33	+
34	+
35	+	def configure(self, cfg_params):
36	+	"""
37	+	Configure the scheduler with the config settings from the user
38	+	"""
39	+
40	+	SchedulerLocal.configure(self, cfg_params)
41	+
42	+	try:
43	+	tmp =  cfg_params['CMSSW.datasetpath']
44	+	if tmp.lower() == 'none':
45	+	self.datasetPath = None
46	+	self.selectNoInput = 1
47	+	else:
48	+	self.datasetPath = tmp
49	+	self.selectNoInput = 0
50	+	except KeyError:
51	+	msg = "Error: datasetpath not defined "
52	+	raise CrabException(msg)
53	+
54	+	self.return_data = cfg_params.get('USER.return_data', 0)
55	+	self.copy_data   = cfg_params.get("USER.copy_data", 0)
56	+
57	+	self.proxyValid = 0
58	+	self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
59	+	self.proxyServer = cfg_params.get("GRID.proxy_server", 'myproxy.cern.ch')
60	+	common.logger.debug('Setting myproxy server to ' + self.proxyServer)
61	+
62	+	self.group = cfg_params.get("GRID.group", None)
63	+	self.role  = cfg_params.get("GRID.role", None)
64	+	self.VO    = cfg_params.get('GRID.virtual_organization', 'cms')
65	+
66	+	self.checkProxy()
67	+
68	+	return
69	+
70	+	def envUniqueID(self):
71	+	taskHash = sha.new(common._db.queryTask('name')).hexdigest()
72	+	id = "https://" + socket.gethostname() + '/' + taskHash + "/${NJob}"
73	+	return id
74	+
75	+	def sched_parameter(self, i, task):
76	+	"""
77	+	Return scheduler-specific parameters
78	+	"""
79	+	req = ''
80	+	if self.EDG_addJdlParam:
81	+	if self.EDG_addJdlParam[-1] == '':
82	+	self.EDG_addJdlParam = self.EDG_addJdlParam[:-1]
83	+	for p in self.EDG_addJdlParam:
84	+	req += p.strip()+';\n'
85	+
86	+	return req
87	+
88	+
89	+	def realSchedParams(self, cfg_params):
90	+	"""
91	+	Return dictionary with specific parameters, to use with real scheduler
92	+	"""
93	+
94	+	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
95	+	tmpDir = os.path.join(common.work_space.shareDir(),'.condor_temp')
96	+	jobDir = common.work_space.jobDir()
97	+	params = {'tmpDir':tmpDir,
98	+	'jobDir':jobDir}
99	+	return params
100	+
101	+
102	+	def listMatch(self, seList, full):
103	+	"""
104	+	Check the compatibility of available resources
105	+	"""
106	+
107	+	return [True]
108	+
109	+
110	+	def decodeLogInfo(self, fileName):
111	+	"""
112	+	Parse logging info file and return main info
113	+	"""
114	+
115	+	import CondorGLoggingInfo
116	+	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
117	+	reason = loggingInfo.decodeReason(fileName)
118	+	return reason
119	+
120	+
121	+	def wsCopyOutput(self):
122	+	"""
123	+	Write a CopyResults part of a job script, e.g.
124	+	to copy produced output into a storage element.
125	+	"""
126	+	txt = self.wsCopyOutput_comm()
127	+	return txt
128	+
129	+
130	+	def wsExitFunc(self):
131	+	"""
132	+	Returns the part of the job script which runs prior to exit
133	+	"""
134	+
135	+	txt = '\n'
136	+	txt += '#\n'
137	+	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
138	+	txt += '#\n\n'
139	+
140	+	txt += 'func_exit() { \n'
141	+	txt += self.wsExitFunc_common()
142	+
143	+	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
144	+	txt += '    cp  ${out_files}.tgz $_CONDOR_SCRATCH_DIR/\n'
145	+	txt += '    cp  crab_fjr_$NJob.xml $_CONDOR_SCRATCH_DIR/\n'
146	+
147	+	txt += '    exit $job_exit_code\n'
148	+	txt += '}\n'
149	+
150	+	return txt
151	+
152	+	def wsInitialEnvironment(self):
153	+	"""
154	+	Returns part of a job script which does scheduler-specific work.
155	+	"""
156	+
157	+	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
158	+	txt += 'echo "Beginning environment"\n'
159	+	txt += 'printenv | sort\n'
160	+
161	+	txt += 'middleware='+self.name()+' \n'
162	+	txt += 'if [ -e /opt/d-cache/srm/bin ]; then\n'
163	+	txt += '  export PATH=${PATH}:/opt/d-cache/srm/bin\n'
164	+	txt += 'fi\n'
165
166	<	# May have problems with onlyOSG being false, probably due to lengths of lists and command line.
73	<	# Either re-write osg_bdii.py with a proper ldap library or break the queries apart
74	<
75	<	#scram = Scram.Scram(None)
76	<	#versionCMSSW = scram.getSWVersion()
77	<	#arch = scram.getArch()
78	<
79	<	if self.selectNoInput:
80	<	return [True]
81	<
82	<
83	<	def decodeLogInfo(self, file):
84	<	"""
85	<	Parse logging info file and return main info
86	<	"""
87	<	import CondorGLoggingInfo
88	<	loggingInfo = CondorGLoggingInfo.CondorGLoggingInfo()
89	<	reason = loggingInfo.decodeReason(file)
90	<	return reason
91	<
92	<
93	<	def wsExitFunc(self):
94	<	"""
95	<	"""
96	<	txt = '\n'
97	<	txt += '#\n'
98	<	txt += '# EXECUTE THIS FUNCTION BEFORE EXIT \n'
99	<	txt += '#\n\n'
100	<
101	<	txt += 'func_exit() { \n'
102	<	txt += self.wsExitFunc_common()
103	<
104	<	txt += '    tar zcvf ${out_files}.tgz  ${final_list}\n'
105	<	txt += '    cp  ${out_files}.tgz $ORIG_WD/\n'
106	<	txt += '    cp  crab_fjr_$NJob.xml $ORIG_WD/\n'
107	<
108	<	txt += '    exit $job_exit_code\n'
109	<	txt += '}\n'
110	<
111	<	return txt
112	<
113	<	def wsInitialEnvironment(self):
114	<	"""
115	<	Returns part of a job script which does scheduler-specific work.
116	<	"""
117	<
118	<	txt  = '\n# Written by SchedulerCondor::wsInitialEnvironment\n'
119	<	txt += 'echo "Beginning environment"\n'
120	<	txt += 'printenv | sort\n'
121	<
122	<	txt += 'middleware='+self.name()+' \n'
123	<	txt += """
166	>	txt += """
167		if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CONDOR_SCRATCH_DIR ]; then
168	<	ORIG_WD=`pwd`
126	<	echo "Change from $ORIG_WD to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
168	>	echo "cd to Condor scratch directory: $_CONDOR_SCRATCH_DIR"
169		if [ -e ../default.tgz ] ;then
170		echo "Found ISB in parent directory (Local Condor)"
171		cp ../default.tgz $_CONDOR_SCRATCH_DIR
#	Line 132 | Line 174 | if [ $_CONDOR_SCRATCH_DIR ] && [ -d $_CO
174		fi
175		"""
176
177	<	return txt
177	>	return txt
178	>
179	>
180	>	def sched_fix_parameter(self):
181	>	"""
182	>	Returns string with requirements and scheduler-specific parameters
183	>	"""
184	>
185	>	if self.EDG_requirements:
186	>	req = self.EDG_requirements
187	>	taskReq = {'commonRequirements':req}
188	>	common._db.updateTask_(taskReq)

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines