CRAB/python/GetOutputServer.py

from GetOutput import GetOutput
from StatusServer import StatusServer
from crab_util import *
import common
import time

import traceback
from xml.dom import minidom
from ServerCommunicator import ServerCommunicator

from ProdCommon.Storage.SEAPI.SElement import SElement
from ProdCommon.Storage.SEAPI.SBinterface import SBinterface

class GetOutputServer( GetOutput, StatusServer ):

    def __init__(self, *args):
 
        GetOutput.__init__(self,*args)

        # init client server params...
        CliServerParams(self)       

        if self.storage_path[0]!='/':
            self.storage_path = '/'+self.storage_path

        return

    
    def getOutput(self): 

        # get updated status from server #inherited from StatusServer
        self.resynchClientSide()

        # understand whether the required output are available
        self.checkBeforeGet()

        # retrive files
        self.retrieveFiles(self.list_id) 

        self.organizeOutput()    

        return

    def retrieveFiles(self,filesToRetrieve):
        """
        Real get output from server storage
        """

        self.taskuuid = str(common._db.queryTask('name'))
        common.logger.debug(3, "Task name: " + self.taskuuid)

        # create the list with the actual filenames 
        remotedir = os.path.join(self.storage_path, self.taskuuid)

      
        # list of file to retrieve
        osbTemplate = remotedir + '/out_files_%s.tgz'  
        osbFiles = [ osbTemplate%str(jid) for jid in filesToRetrieve ]
        common.logger.debug(3, "List of OSB files: " +str(osbFiles) )
 
        #   
        copyHere = common.work_space.resDir()
        destTemplate = copyHere+'/out_files_%s.tgz'  
        destFiles = [ destTemplate%str(jid) for jid in filesToRetrieve ]

        common.logger.message("Starting retrieving output from server "+str(self.storage_name)+"...")

        try:  
            seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
        except Exception, ex:
            common.logger.debug(1, str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            raise CrabException(msg)
        try:  
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(1, str(ex))
            msg = "ERROR : Unable to create destination  interface \n"
            raise CrabException(msg)

        ## copy ISB ##
        sbi = SBinterface( seEl, loc )

        # retrieve them from SE #TODO replace this with SE-API
        for i in xrange(len(filesToRetrieve)):
            source = osbFiles[i] 
            dest = destFiles[i]
            common.logger.debug(1, "retrieving "+ str(source) +" to "+ str(dest) )
            try:
                sbi.copy( source, dest)
            except Exception, ex:
                msg = "WARNING: Unable to retrieve output file %s \n"%osbFiles[i] 
                msg += str(ex)
                common.logger.debug(1,msg)
                continue

        # notify to the server that output have been retrieved successfully. proxy from StatusServer
        if len(filesToRetrieve) > 0:
            common.logger.debug(5, "List of retrieved files notified to server: %s"%str(filesToRetrieve) ) 
            csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params)
            csCommunicator.outputRetrieved(self.taskuuid, filesToRetrieve)

        return


Revision:	1.22
Committed:	Sat Apr 26 08:25:00 2008 UTC (17 years ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_2_0_pre13
Changes since 1.21:	+4 -29 lines
Log Message:	Correct proxy management....not safe to check it everywhere. Added support for caf (e.g. local ) server. Code reorganization..
#	User	Rev	Content
1	spiga	1.16	from GetOutput import GetOutput
2			from StatusServer import StatusServer
3	spiga	1.1	from crab_util import *
4			import common
5			import time
6
7	farinafa	1.14	import traceback
8			from xml.dom import minidom
9			from ServerCommunicator import ServerCommunicator
10
11	spiga	1.19	from ProdCommon.Storage.SEAPI.SElement import SElement
12			from ProdCommon.Storage.SEAPI.SBinterface import SBinterface
13	farinafa	1.14
14	spiga	1.18	class GetOutputServer( GetOutput, StatusServer ):
15	farinafa	1.14
16			def __init__(self, *args):
17
18	spiga	1.21	GetOutput.__init__(self,*args)
19
20	spiga	1.22	# init client server params...
21			CliServerParams(self)
22	farinafa	1.14
23			if self.storage_path[0]!='/':
24			self.storage_path = '/'+self.storage_path
25	spiga	1.16
26	spiga	1.1	return
27	farinafa	1.14
28	spiga	1.18
29			def getOutput(self):
30	spiga	1.1
31	farinafa	1.14	# get updated status from server #inherited from StatusServer
32			self.resynchClientSide()
33	mcinquil	1.2
34	farinafa	1.14	# understand whether the required output are available
35			self.checkBeforeGet()
36	spiga	1.18
37			# retrive files
38			self.retrieveFiles(self.list_id)
39
40			self.organizeOutput()
41
42			return
43
44			def retrieveFiles(self,filesToRetrieve):
45			"""
46			Real get output from server storage
47			"""
48	spiga	1.19
49			self.taskuuid = str(common._db.queryTask('name'))
50			common.logger.debug(3, "Task name: " + self.taskuuid)
51	farinafa	1.14
52			# create the list with the actual filenames
53	spiga	1.19	remotedir = os.path.join(self.storage_path, self.taskuuid)
54	farinafa	1.14
55	spiga	1.19
56			# list of file to retrieve
57			osbTemplate = remotedir + '/out_files_%s.tgz'
58	farinafa	1.14	osbFiles = [ osbTemplate%str(jid) for jid in filesToRetrieve ]
59	spiga	1.19	common.logger.debug(3, "List of OSB files: " +str(osbFiles) )
60
61			#
62	slacapra	1.17	copyHere = common.work_space.resDir()
63	spiga	1.19	destTemplate = copyHere+'/out_files_%s.tgz'
64	farinafa	1.14	destFiles = [ destTemplate%str(jid) for jid in filesToRetrieve ]
65
66	spiga	1.19	common.logger.message("Starting retrieving output from server "+str(self.storage_name)+"...")
67
68			try:
69			seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
70			except Exception, ex:
71			common.logger.debug(1, str(ex))
72			msg = "ERROR : Unable to create SE source interface \n"
73			raise CrabException(msg)
74			try:
75			loc = SElement("localhost", "local")
76			except Exception, ex:
77			common.logger.debug(1, str(ex))
78			msg = "ERROR : Unable to create destination interface \n"
79			raise CrabException(msg)
80
81			## copy ISB ##
82			sbi = SBinterface( seEl, loc )
83
84	farinafa	1.14	# retrieve them from SE #TODO replace this with SE-API
85			for i in xrange(len(filesToRetrieve)):
86	spiga	1.19	source = osbFiles[i]
87			dest = destFiles[i]
88			common.logger.debug(1, "retrieving "+ str(source) +" to "+ str(dest) )
89	farinafa	1.15	try:
90	spiga	1.22	sbi.copy( source, dest)
91	spiga	1.19	except Exception, ex:
92	spiga	1.20	msg = "WARNING: Unable to retrieve output file %s \n"%osbFiles[i]
93			msg += str(ex)
94			common.logger.debug(1,msg)
95	farinafa	1.15	continue
96	farinafa	1.14
97			# notify to the server that output have been retrieved successfully. proxy from StatusServer
98	farinafa	1.15	if len(filesToRetrieve) > 0:
99			common.logger.debug(5, "List of retrieved files notified to server: %s"%str(filesToRetrieve) )
100	spiga	1.22	csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params)
101	spiga	1.19	csCommunicator.outputRetrieved(self.taskuuid, filesToRetrieve)
102	spiga	1.1
103	farinafa	1.14	return
104	spiga	1.1
105