ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerEdg.py

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.73.2.2 by fanzago, Wed Jul 19 10:53:40 2006 UTC vs.
Revision 1.73.2.3 by spiga, Wed Jul 19 14:57:09 2006 UTC

#	Line 21 | Line 21 | class SchedulerEdg(Scheduler):
21		return
22
23		def configure(self, cfg_params):
24	<	# test branch 2
24	>
25		try:
26		RB = cfg_params["EDG.rb"]
27		edgConfig = EdgConfig(RB)
#	Line 48 | Line 48 | class SchedulerEdg(Scheduler):
48
49		try:
50		self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
51	–	#print "self.EDG_ce_black_list = ", self.EDG_ce_black_list
51		except KeyError:
52		self.EDG_ce_black_list  = ''
53
54		try:
55		self.EDG_ce_white_list = cfg_params['EDG.ce_white_list']
57	–	#print "self.EDG_ce_white_list = ", self.EDG_ce_white_list
56		except KeyError: self.EDG_ce_white_list = ''
57
58		try: self.VO = cfg_params['EDG.virtual_organization']
#	Line 65 | Line 63 | class SchedulerEdg(Scheduler):
63
64		try:
65		self.copy_input_data = common.analisys_common_info['copy_input_data']
68	–	#print "self.copy_input_data = ", self.copy_input_data
66		except KeyError: self.copy_input_data = 0
67
68		try:
#	Line 139 | Line 136 | class SchedulerEdg(Scheduler):
136		except KeyError: self.EDG_cpu_time = ''
137
138		# Add EDG_WL_LOCATION to the python path
142	–
139		try:
140		path = os.environ['EDG_WL_LOCATION']
141		except:
#	Line 165 | Line 161 | class SchedulerEdg(Scheduler):
161		"""
162		Returns file with scheduler-specific parameters
163		"""
168	–
164		if (self.edg_config and self.edg_config_vo != ''):
165		self.param='sched_param.clad'
166		param_file = open(common.work_space.shareDir()+'/'+self.param, 'w')
#	Line 690 | Line 685 | class SchedulerEdg(Scheduler):
685		cmd_out = runCommand(cmd)
686		return cmd_out
687
688	<	def createSchScript(self, nj):
688	>
689	>	def createXMLSchScript(self, nj):
690		"""
691	<	Create a JDL-file for EDG.
691	>	Create a XML-file for BOSS4.
692		"""
697	–
693		job = common.job_list[nj]
694		jbt = job.type()
695		inp_sandbox = jbt.inputSandbox(nj)
696		out_sandbox = jbt.outputSandbox(nj)
702	–	inp_storage_subdir = ''
697
698	<	title = '# This JDL was generated by '+\
705	<	common.prog_name+' (version '+common.prog_version_str+')\n'
698	>	title = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n'
699		jt_string = ''
700	+
701	+	xml_fname = 'orca.xml'
702	+	xml = open(common.work_space.shareDir()+'/'+xml_fname, 'a')
703
704	<
704	>	#TaskName
705	>	dir = string.split(common.work_space.topDir(), '/')
706	>	taskName = dir[len(dir)-2]
707	>
708	>	to_writeReq = ''
709	>	to_write = ''
710	>	req=' '
711	>	req = req + jbt.getRequirements()
712	>	if self.EDG_requirements:
713	>	if (req == ' '):
714	>	req = req + self.EDG_requirements
715	>	else:
716	>	req = req +  ' && ' + self.EDG_requirements
717	>	if self.EDG_ce_white_list:
718	>	ce_white_list = string.split(self.EDG_ce_white_list,',')
719	>	for i in range(len(ce_white_list)):
720	>	if i == 0:
721	>	if (req == ' '):
722	>	req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
723	>	else:
724	>	req = req +  ' && ((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
725	>	pass
726	>	else:
727	>	req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
728	>	req = req + ')'
729
730	<	SPL = inp_storage_subdir
731	<	if ( SPL and SPL[-1] != '/' ) : SPL = SPL + '/'
730	>	if self.EDG_ce_black_list:
731	>	ce_black_list = string.split(self.EDG_ce_black_list,',')
732	>	for ce in ce_black_list:
733	>	if (req == ' '):
734	>	req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
735	>	else:
736	>	req = req +  ' && (!RegExp("' + ce + '", other.GlueCEUniqueId))'
737	>	pass
738	>	if self.EDG_clock_time:
739	>	if (req == ' '):
740	>	req = req + 'other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
741	>	else:
742	>	req = req + ' && other.GlueCEPolicyMaxWallClockTime>='+self.EDG_clock_time
743	>
744	>	if self.EDG_cpu_time:
745	>	if (req == ' '):
746	>	req = req + ' other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
747	>	else:
748	>	req = req + ' && other.GlueCEPolicyMaxCPUTime>='+self.EDG_cpu_time
749	>	if (req != ' '):
750	>	req = req + '\n'
751	>	to_writeReq = req
752	>
753	>
754	>	if ( self.EDG_retry_count ):
755	>	to_write = to_write + 'RetryCount = "'+self.EDG_retry_count+'"\n'
756	>	pass
757
758	<	jdl_fname = job.jdlFilename()
759	<	jdl = open(jdl_fname, 'w')
715	<	jdl.write(title)
758	>	to_write = to_write + 'MyProxyServer = "&quot;' + self.proxyServer + '&quot;"\n'
759	>	to_write = to_write + 'VirtualOrganisation = "&quot;' + self.VO + '&quot;"\n'
760
717	–	script = job.scriptFilename()
718	–	jdl.write('Executable = "' + os.path.basename(script) +'";\n')
719	–	jdl.write(jt_string)
761
762	<	### only one .sh  JDL has arguments:
762	>	#TaskName
763	>	dir = string.split(common.work_space.topDir(), '/')
764	>	taskName = dir[len(dir)-2]
765	>
766	>	if nj == 0:
767	>	xml.write(str(title))
768	>	xml.write('<task name="' +str(taskName)+'">\n')
769	>	xml.write(jt_string)
770	>	#Here it must pass the extra Tags.. (into Task & out of chain)
771	>	if (to_writeReq != ''):
772	>	xml.write('<extraTags>\n')
773	>	xml.write('<Requirements>\n')
774	>	xml.write('<![CDATA[\n')
775	>	xml.write(to_writeReq)
776	>	xml.write(']]>\n')
777	>	xml.write('</Requirements>\n')
778	>	xml.write('</extraTags>\n')
779	>	pass
780
781	+	if (to_write != ''):
782	+	xml.write('<extraTags\n')
783	+	xml.write(to_write)
784	+	xml.write('/>\n')
785	+	pass
786	+	pass
787	+
788	+	xml.write('<chain scheduler="edg">\n')
789	+	xml.write(jt_string)
790	+
791	+	#executable
792	+	script = job.scriptFilename()
793	+	xml.write('<program exec="' + os.path.basename(script) +'"\n')
794	+	xml.write(jt_string)
795	+
796	+
797	+	### only one .sh  JDL has arguments:
798		### Fabio
799	<	jdl.write('Arguments = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'";\n')
800	<	inp_box = 'InputSandbox = { '
801	<	inp_box = inp_box + '"' + script + '",'
799	>	xml.write('args = "' + str(nj+1)+' '+ jbt.getJobTypeArguments(nj, "EDG") +'"\n')
800	>	xml.write('program_types="crabjob"\n')
801	>	inp_box = 'infiles="'
802	>	inp_box = inp_box + '' + script + ','
803
804		if inp_sandbox != None:
805		for fl in inp_sandbox:
806	<	inp_box = inp_box + ' "' + fl + '",'
806	>	inp_box = inp_box + '' + fl + ','
807		pass
808		pass
809
734	–	#if common.use_jam:
735	–	#   inp_box = inp_box+' "'+common.bin_dir+'/'+common.run_jam+'",'
736	–
810		# Marco (VERY TEMPORARY ML STUFF)
811	<	inp_box = inp_box+' "' + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + '", "' +\
812	<	os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + '", "'+\
813	<	os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + '", "'+\
814	<	os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + '", "'+\
815	<	os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py') + '"'
811	>	inp_box = inp_box + os.path.abspath(os.environ['CRABDIR']+'/python/'+'report.py') + ',' +\
812	>	os.path.abspath(os.environ['CRABDIR']+'/python/'+'DashboardAPI.py') + ','+\
813	>	os.path.abspath(os.environ['CRABDIR']+'/python/'+'Logger.py') + ','+\
814	>	os.path.abspath(os.environ['CRABDIR']+'/python/'+'ProcInfo.py') + ','+\
815	>	os.path.abspath(os.environ['CRABDIR']+'/python/'+'apmon.py')
816		# End Marco
817
818		if (not jbt.additional_inbox_files == []):
819		inp_box = inp_box + ', '
820		for addFile in jbt.additional_inbox_files:
821		addFile = os.path.abspath(addFile)
822	<	inp_box = inp_box+' "'+addFile+'",'
822	>	inp_box = inp_box+''+addFile+','
823		pass
824
825		if inp_box[-1] == ',' : inp_box = inp_box[:-1]
826	<	inp_box = inp_box + ' };\n'
827	<	jdl.write(inp_box)
826	>	inp_box = inp_box + ' "\n'
827	>	xml.write(inp_box)
828
829	<	jdl.write('StdOutput     = "' + job.stdout() + '";\n')
830	<	jdl.write('StdError      = "' + job.stderr() + '";\n')
829	>	xml.write('stderr="' + job.stdout() + '"\n')
830	>	xml.write('stdout="' + job.stderr() + '"\n')
831
832
833		if job.stdout() == job.stderr():
834	<	out_box = 'OutputSandbox = { "' + \
835	<	job.stdout() + '", ".BrokerInfo",'
834	>	out_box = 'outfiles="' + \
835	>	job.stdout() + ',.BrokerInfo,'
836		else:
837	<	out_box = 'OutputSandbox = { "' + \
838	<	job.stdout() + '", "' + \
839	<	job.stderr() + '", ".BrokerInfo",'
837	>	out_box = 'outfiles="' + \
838	>	job.stdout() + ',' + \
839	>	job.stderr() + ',.BrokerInfo,'
840
841		if int(self.return_data) == 1:
842		if out_sandbox != None:
843		for fl in out_sandbox:
844	<	out_box = out_box + ' "' + fl + '",'
844	>	out_box = out_box + '' + fl + ','
845		pass
846		pass
847		pass
848
849		if out_box[-1] == ',' : out_box = out_box[:-1]
850	<	out_box = out_box + ' };'
851	<	jdl.write(out_box+'\n')
850	>	out_box = out_box + '"'
851	>	xml.write(out_box+'\n')
852	>
853	>	xml.write('group="'+taskName+'"\n')
854	>	xml.write('BossAttr="[crabjob.INTERNAL_ID=' + str(nj+1) +']"\n')
855
780	–
781	–	req='Requirements = '
782	–	noreq=req
783	–	req = req + jbt.getRequirements()
784	–	#### and USER REQUIREMENT
785	–	if self.EDG_requirements:
786	–	if (req != noreq):
787	–	req = req +  ' && '
788	–	req = req + self.EDG_requirements
789	–	#### FEDE #####
790	–	if self.EDG_ce_white_list:
791	–	ce_white_list = string.split(self.EDG_ce_white_list,',')
792	–	#print "req = ", req
793	–	for i in range(len(ce_white_list)):
794	–	if i == 0:
795	–	if (req != noreq):
796	–	req = req +  ' && '
797	–	req = req + '((RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
798	–	pass
799	–	else:
800	–	req = req +  ' || (RegExp("' + ce_white_list[i] + '", other.GlueCEUniqueId))'
801	–	req = req + ')'
856
857	<	if self.EDG_ce_black_list:
858	<	ce_black_list = string.split(self.EDG_ce_black_list,',')
859	<	for ce in ce_black_list:
806	<	if (req != noreq):
807	<	req = req +  ' && '
808	<	req = req + '(!RegExp("' + ce + '", other.GlueCEUniqueId))'
809	<	pass
810	<
811	<	###############
812	<	clockTime=480
813	<	if self.EDG_clock_time:
814	<	clockTime= self.EDG_clock_time
815	<	if (req != noreq):
816	<	req = req + ' && '
817	<	req = req + '((other.GlueCEPolicyMaxWallClockTime == 0) || (other.GlueCEPolicyMaxWallClockTime>='+str(clockTime)+'))'
857	>
858	>	xml.write('/>\n')
859	>	xml.write('</chain>\n')
860
861	<	cpuTime=1000
820	<	if self.EDG_cpu_time:
821	<	cpuTime=self.EDG_cpu_time
822	<	if (req != noreq):
823	<	req = req + ' && '
824	<	req = req + '((other.GlueCEPolicyMaxCPUTime == 0) || (other.GlueCEPolicyMaxCPUTime>='+str(cpuTime)+'))'
825	<
826	<	if (req != noreq):
827	<	req = req + ';\n'
828	<	jdl.write(req)
829	<
830	<	jdl.write('VirtualOrganisation = "' + self.VO + '";\n')
861	>	if int(nj) == int(common.jobDB.nJobs()-1): xml.write('</task>\n')
862
832	–	if ( self.EDG_retry_count ):
833	–	jdl.write('RetryCount = '+self.EDG_retry_count+';\n')
834	–	pass
863
864	<	jdl.write('MyProxyServer = "' + self.proxyServer + '";\n')
864	>	xml.close()
865
838	–	jdl.close()
866		return
867
868	+
869	+
870		def checkProxy(self):
871		"""
872		Function to check the Globus proxy.

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines