ViewVC Help

View File | Revision Log | Show Annotations | Root Listing

root/cvsroot/COMP/CRAB/python/SchedulerEdg.py

Comparing COMP/CRAB/python/SchedulerEdg.py (file contents):
Revision 1.133.2.4 by fanzago, Fri Oct 12 12:56:30 2007 UTC vs.
Revision 1.137 by spiga, Tue Sep 18 15:17:55 2007 UTC

#	Line 52 | Line 52 | class SchedulerEdg(Scheduler):
52		except KeyError:
53		self.role = None
54
55	<	#try: self.LCG_version = cfg_params["EDG.lcg_version"]
56	<	#except KeyError: self.LCG_version = '2'
55	>	try: self.LCG_version = cfg_params["EDG.lcg_version"]
56	>	except KeyError: self.LCG_version = '2'
57
58		try:
59		self.EDG_ce_black_list = cfg_params['EDG.ce_black_list']
#	Line 87 | Line 87 | class SchedulerEdg(Scheduler):
87		except KeyError: self.copy_data = 0
88
89		if ( int(self.return_data) == 0 and int(self.copy_data) == 0 ):
90	<	msg = 'Error: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n'
90	>	msg = 'Warning: return_data = 0 and copy_data = 0 ==> your exe output will be lost\n'
91		msg = msg + 'Please modify return_data and copy_data value in your crab.cfg file\n'
92		raise CrabException(msg)
93
94	–	if ( int(self.return_data) == 1 and int(self.copy_data) == 1 ):
95	–	msg = 'Error: return_data and copy_data cannot be set both to 1\n'
96	–	msg = msg + 'Please modify return_data or copy_data value in your crab.cfg file\n'
97	–	raise CrabException(msg)
98	–
94		########### FEDE FOR DBS2 ##############################
95		try:
96		self.publish_data = cfg_params["USER.publish_data"]
#	Line 127 | Line 122 | class SchedulerEdg(Scheduler):
122		raise CrabException(msg)
123		#################################################
124
125	<	#try:
126	<	#    self.lfc_host = cfg_params['EDG.lfc_host']
127	<	#except KeyError:
128	<	#    msg = "Error. The [EDG] section does not have 'lfc_host' value"
129	<	#    msg = msg + " it's necessary to know the LFC host name"
130	<	#    common.logger.message(msg)
131	<	#    raise CrabException(msg)
132	<	#try:
133	<	#    self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
134	<	#except KeyError:
135	<	#    msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
136	<	#    msg = msg + " it's necessary to know the catalog type"
137	<	#    common.logger.message(msg)
138	<	#    raise CrabException(msg)
139	<	#try:
140	<	#    self.lfc_home = cfg_params['EDG.lfc_home']
141	<	#except KeyError:
142	<	#    msg = "Error. The [EDG] section does not have 'lfc_home' value"
143	<	#    msg = msg + " it's necessary to know the home catalog dir"
144	<	#    common.logger.message(msg)
145	<	#    raise CrabException(msg)
125	>	try:
126	>	self.lfc_host = cfg_params['EDG.lfc_host']
127	>	except KeyError:
128	>	msg = "Error. The [EDG] section does not have 'lfc_host' value"
129	>	msg = msg + " it's necessary to know the LFC host name"
130	>	common.logger.message(msg)
131	>	raise CrabException(msg)
132	>	try:
133	>	self.lcg_catalog_type = cfg_params['EDG.lcg_catalog_type']
134	>	except KeyError:
135	>	msg = "Error. The [EDG] section does not have 'lcg_catalog_type' value"
136	>	msg = msg + " it's necessary to know the catalog type"
137	>	common.logger.message(msg)
138	>	raise CrabException(msg)
139	>	try:
140	>	self.lfc_home = cfg_params['EDG.lfc_home']
141	>	except KeyError:
142	>	msg = "Error. The [EDG] section does not have 'lfc_home' value"
143	>	msg = msg + " it's necessary to know the home catalog dir"
144	>	common.logger.message(msg)
145	>	raise CrabException(msg)
146
147	<	#try:
148	<	#    self.register_data = cfg_params["USER.register_data"]
149	<	#    if int(self.register_data) == 1:
150	<	#        try:
151	<	#            self.LFN = cfg_params['USER.lfn_dir']
152	<	#        except KeyError:
153	<	#            msg = "Error. The [USER] section does not have 'lfn_dir' value"
154	<	#            msg = msg + " it's necessary for LCF registration"
155	<	#            common.logger.message(msg)
156	<	#            raise CrabException(msg)
157	<	#except KeyError: self.register_data = 0
158	<
159	<	#if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
160	<	#   msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
161	<	#   msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
162	<	#   common.logger.message(msg)
163	<	#   raise CrabException(msg)
147	>	try:
148	>	self.register_data = cfg_params["USER.register_data"]
149	>	if int(self.register_data) == 1:
150	>	try:
151	>	self.LFN = cfg_params['USER.lfn_dir']
152	>	except KeyError:
153	>	msg = "Error. The [USER] section does not have 'lfn_dir' value"
154	>	msg = msg + " it's necessary for LCF registration"
155	>	common.logger.message(msg)
156	>	raise CrabException(msg)
157	>	except KeyError: self.register_data = 0
158	>
159	>	if ( int(self.copy_data) == 0 and int(self.register_data) == 1 ):
160	>	msg = 'Warning: register_data = 1 must be used with copy_data = 1\n'
161	>	msg = msg + 'Please modify copy_data value in your crab.cfg file\n'
162	>	common.logger.message(msg)
163	>	raise CrabException(msg)
164
165		try: self.EDG_requirements = cfg_params['EDG.requirements']
166		except KeyError: self.EDG_requirements = ''
#	Line 330 | Line 325 | class SchedulerEdg(Scheduler):
325		txt += 'echo "middleware discovery " \n'
326		txt += 'if [ $GRID3_APP_DIR ]; then\n'
327		txt += '    middleware=OSG \n'
328	<	txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
334	<	txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
335	<	txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
336	<	txt += '    else\n'
337	<	txt += '        echo "not reporting SyncCE";\n'
338	<	txt += '    fi\n';
328	>	txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
329		txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
330		txt += '    echo "middleware =$middleware" \n'
331		txt += 'elif [ $OSG_APP ]; then \n'
332		txt += '    middleware=OSG \n'
333	<	txt += '    if [ $GLOBUS_GRAM_JOB_CONTACT ]; then \n'
344	<	txt += '        SyncCE=`echo "echo $GLOBUS_GRAM_JOB_CONTACT" | cut -d: -f2 | sed \'s/\/\///\'`;\n'
345	<	txt += '        echo "SyncCE=$SyncCE"  | tee -a $RUNTIME_AREA/$repo ;\n'
346	<	txt += '    else\n'
347	<	txt += '        echo "not reporting SyncCE";\n'
348	<	txt += '    fi\n';
333	>	txt += '    echo "SyncCE=`echo $EDG_WL_LOG_DESTINATION`" | tee -a $RUNTIME_AREA/$repo \n'
334		txt += '    echo "GridFlavour=`echo $middleware`" | tee -a $RUNTIME_AREA/$repo \n'
335		txt += '    echo "middleware =$middleware" \n'
336		txt += 'elif [ $VO_CMS_SW_DIR ]; then \n'
#	Line 384 | Line 369 | class SchedulerEdg(Scheduler):
369
370		txt += 'export VO='+self.VO+'\n'
371		### add some line for LFC catalog setting
372	<	#txt += 'if [ $middleware == LCG ]; then \n'
373	<	#txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
374	<	#txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
375	<	#txt += '    fi\n'
376	<	#txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
377	<	#txt += '        export LFC_HOST='+self.lfc_host+'\n'
378	<	#txt += '    fi\n'
379	<	#txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
380	<	#txt += '        export LFC_HOME='+self.lfc_home+'\n'
381	<	#txt += '    fi\n'
382	<	#txt += 'elif [ $middleware == OSG ]; then\n'
383	<	#txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
384	<	#txt += 'fi\n'
372	>	txt += 'if [ $middleware == LCG ]; then \n'
373	>	txt += '    if [[ $LCG_CATALOG_TYPE != \''+self.lcg_catalog_type+'\' ]]; then\n'
374	>	txt += '        export LCG_CATALOG_TYPE='+self.lcg_catalog_type+'\n'
375	>	txt += '    fi\n'
376	>	txt += '    if [[ $LFC_HOST != \''+self.lfc_host+'\' ]]; then\n'
377	>	txt += '        export LFC_HOST='+self.lfc_host+'\n'
378	>	txt += '    fi\n'
379	>	txt += '    if [[ $LFC_HOME != \''+self.lfc_home+'\' ]]; then\n'
380	>	txt += '        export LFC_HOME='+self.lfc_home+'\n'
381	>	txt += '    fi\n'
382	>	txt += 'elif [ $middleware == OSG ]; then\n'
383	>	txt += '    echo "LFC catalog setting to be implemented for OSG"\n'
384	>	txt += 'fi\n'
385		#####
386	<	#if int(self.register_data) == 1:
387	<	#   txt += 'if [ $middleware == LCG ]; then \n'
388	<	#   txt += '    export LFN='+self.LFN+'\n'
389	<	#   txt += '    lfc-ls $LFN\n'
390	<	#   txt += '    result=$?\n'
391	<	#   txt += '    echo $result\n'
392	<	#   ### creation of LFN dir in LFC catalog, under /grid/cms dir
393	<	#   txt += '    if [ $result != 0 ]; then\n'
394	<	#   txt += '       lfc-mkdir $LFN\n'
395	<	#   txt += '       result=$?\n'
396	<	#   txt += '       echo $result\n'
397	<	#   txt += '    fi\n'
398	<	#   txt += 'elif [ $middleware == OSG ]; then\n'
399	<	#   txt += '    echo " Files registration to be implemented for OSG"\n'
400	<	#   txt += 'fi\n'
401	<	#   txt += '\n'
402	<	#   if self.VO:
403	<	#      txt += 'export VO='+self.VO+'\n'
404	<	#   if self.LFN:
405	<	#      txt += 'if [ $middleware == LCG ]; then \n'
406	<	#      txt += '    export LFN='+self.LFN+'\n'
407	<	#      txt += 'fi\n'
408	<	#      txt += '\n'
386	>	if int(self.register_data) == 1:
387	>	txt += 'if [ $middleware == LCG ]; then \n'
388	>	txt += '    export LFN='+self.LFN+'\n'
389	>	txt += '    lfc-ls $LFN\n'
390	>	txt += '    result=$?\n'
391	>	txt += '    echo $result\n'
392	>	### creation of LFN dir in LFC catalog, under /grid/cms dir
393	>	txt += '    if [ $result != 0 ]; then\n'
394	>	txt += '       lfc-mkdir $LFN\n'
395	>	txt += '       result=$?\n'
396	>	txt += '       echo $result\n'
397	>	txt += '    fi\n'
398	>	txt += 'elif [ $middleware == OSG ]; then\n'
399	>	txt += '    echo " Files registration to be implemented for OSG"\n'
400	>	txt += 'fi\n'
401	>	txt += '\n'
402	>
403	>	if self.VO:
404	>	txt += 'export VO='+self.VO+'\n'
405	>	if self.LFN:
406	>	txt += 'if [ $middleware == LCG ]; then \n'
407	>	txt += '    export LFN='+self.LFN+'\n'
408	>	txt += 'fi\n'
409	>	txt += '\n'
410
411		txt += 'if [ $middleware == LCG ]; then\n'
412		#    txt += '    CloseCEs=`edg-brokerinfo getCE`\n'
#	Line 523 | Line 509 | class SchedulerEdg(Scheduler):
509		txt += 'echo "# Copy output files from WN = `hostname` to SE = $SE"\n'
510		txt += 'echo "####################################################"\n'
511
512	<	txt += 'if [ $output_exit_status -eq 60302 ]; then\n'
513	<	txt += '    echo "--> No output file to copy to $SE"\n'
514	<	txt += '    copy_exit_status=$output_exit_status\n'
515	<	txt += '    echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
516	<	txt += 'else\n'
517	<	txt += '    for out_file in $file_list ; do\n'
518	<	txt += '        echo "Trying to copy output file to $SE"\n'
519	<	txt += '        cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
520	<	txt += '        copy_exit_status=$?\n'
521	<	txt += '        echo "COPY_EXIT_STATUS = $copy_exit_status"\n'
522	<	txt += '        echo "STAGE_OUT = $copy_exit_status"\n'
523	<	txt += '        if [ $copy_exit_status -ne 0 ]; then\n'
524	<	txt += '            echo "Problem copying $out_file to $SE $SE_PATH"\n'
525	<	txt += '            echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
526	<	txt += '            copy_exit_status=60307\n'
527	<	txt += '        else\n'
528	<	txt += '            echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
529	<	txt += '            echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
530	<	txt += '            echo "output copied into $SE/$SE_PATH directory"\n'
545	<	txt += '            echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
546	<	txt += '        fi\n'
547	<	txt += '    done\n'
548	<	txt += '    if [ $copy_exit_status -ne 0 ]; then\n'
549	<	txt += '        SE=""\n'
550	<	txt += '        echo "SE = $SE"\n'
551	<	txt += '        SE_PATH=""\n'
552	<	txt += '        echo "SE_PATH = $SE_PATH"\n'
512	>	txt += 'for out_file in $file_list ; do\n'
513	>	txt += '   echo "Trying to copy output file to $SE using srmcp"\n'
514	>	txt += '   cmscp $out_file ${SE} ${SE_PATH} $out_file $middleware\n'
515	>	txt += '   copy_exit_status=$?\n'
516	>	txt += '   echo "COPY_EXIT_STATUS for srmcp = $copy_exit_status"\n'
517	>	txt += '   echo "STAGE_OUT = $copy_exit_status"\n'
518	>
519	>	txt += '   if [ $copy_exit_status -ne 0 ]; then\n'
520	>	txt += '       echo "Problem copying $out_file to $SE $SE_PATH"\n'
521	>	txt += '       echo "StageOutExitStatus = $copy_exit_status " | tee -a $RUNTIME_AREA/$repo\n'
522	>	txt += '       echo "StageOutExitStatusReason = $exitstring" | tee -a $RUNTIME_AREA/$repo\n'
523	>	txt += '       copy_exit_status=60307\n'
524	>
525	>	txt += '   else\n'
526	>	txt += '       echo "StageOutSE = $SE" | tee -a $RUNTIME_AREA/$repo\n'
527	>	txt += '       echo "StageOutCatalog = " | tee -a $RUNTIME_AREA/$repo\n'
528	>	txt += '       echo "output copied into $SE/$SE_PATH directory"\n'
529	>	txt += '       echo "StageOutExitStatus = 0" | tee -a $RUNTIME_AREA/$repo\n'
530	>	txt += '       echo "srmcp succeeded"\n'
531		txt += '    fi\n'
532	+	txt += 'done\n'
533	+	txt += 'if [ $copy_exit_status -ne 0 ]; then\n'
534	+	txt += '      SE=""\n'
535	+	txt += '      echo "SE = $SE"\n'
536	+	txt += '      SE_PATH=""\n'
537	+	txt += '      echo "SE_PATH = $SE_PATH"\n'
538		txt += 'fi\n'
539		txt += 'exit_status=$copy_exit_status\n'
540		pass
#	Line 587 | Line 571 | class SchedulerEdg(Scheduler):
571		replicas = self.blackWhiteListParser.checkWhiteList(replicas,n)
572
573		if len(replicas)==0:
574	<	itr = itr + 'target.GlueSEUniqueID=="NONE" '
575	<	#msg = 'No sites remaining that host any part of the requested data! Exiting... '
592	<	#raise CrabException(msg)
574	>	msg = 'No sites remaining that host any part of the requested data! Exiting... '
575	>	raise CrabException(msg)
576		#####
577		# for site in sites:
578		for site in replicas:
#	Line 868 | Line 851 | class SchedulerEdg(Scheduler):
851		common.logger.message('No credential delegated to myproxy server '+self.proxyServer+' will do now')
852		renewProxy = 1
853		else:
854	<	## minimum time: 5 days
855	<	minTime = 4 * 24 * 3600
856	<	## regex to extract the right information
857	<	myproxyRE = re.compile("timeleft: (?P<hours>[\\d]*):(?P<minutes>[\\d]*):(?P<seconds>[\\d]*)")
858	<	for row in cmd_out.split("\n"):
859	<	g = myproxyRE.search(row)
860	<	if g:
861	<	hours = g.group("hours")
879	<	minutes = g.group("minutes")
880	<	seconds = g.group("seconds")
881	<	timeleft = int(hours)*3600 + int(minutes)*60 + int(seconds)
882	<	if timeleft < minTime:
883	<	renewProxy = 1
884	<	common.logger.message('Your proxy will expire in:\n\t'+hours+' hours '+minutes+' minutes '+seconds+' seconds\n')
885	<	common.logger.message('Need to renew it:')
886	<	pass
854	>	# if myproxy exist but not long enough, renew
855	>	reTime = re.compile( r'timeleft: (\d+)' )
856	>	#print "<"+str(reTime.search( cmd_out ).group(1))+">"
857	>	if reTime.match( cmd_out ):
858	>	time = reTime.search( cmd_out ).group(1)
859	>	if time < minTimeLeftServer:
860	>	renewProxy = 1
861	>	common.logger.message('No credential delegation will expire in '+time+' hours: renew it')
862		pass
863		pass
864

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines