CRAB/python/GetOutputServer.py

from GetOutput import GetOutput
from StatusServer import StatusServer
from crab_util import *
import common
import time

import traceback
from xml.dom import minidom
from ServerCommunicator import ServerCommunicator

from ProdCommon.Storage.SEAPI.SElement import SElement
from ProdCommon.Storage.SEAPI.SBinterface import SBinterface

class GetOutputServer( GetOutput, StatusServer ):

    def __init__(self, *args):
 
        GetOutput.__init__(self,*args)

        # init client server params...
        CliServerParams(self)       

        if self.storage_path[0]!='/':
            self.storage_path = '/'+self.storage_path

        return

    
    def getOutput(self): 

        # get updated status from server #inherited from StatusServer
        self.resynchClientSide()

        # understand whether the required output are available
        self.checkBeforeGet()

        # retrive files
        self.retrieveFiles(self.list_id) 

        self.organizeOutput()    

        return

    def retrieveFiles(self,filesToRetrieve):
        """
        Real get output from server storage
        """

        self.taskuuid = str(common._db.queryTask('name'))
        common.logger.debug(3, "Task name: " + self.taskuuid)

        # create the list with the actual filenames 
        remotedir = os.path.join(self.storage_path, self.taskuuid)

      
        # list of file to retrieve
        osbTemplate = remotedir + '/out_files_%s.tgz'  
        osbFiles = [ osbTemplate%str(jid) for jid in filesToRetrieve ]
        common.logger.debug(3, "List of OSB files: " +str(osbFiles) )
 
        #   
        copyHere = self.outDir 
        destTemplate = copyHere+'/out_files_%s.tgz'  
        destFiles = [ destTemplate%str(jid) for jid in filesToRetrieve ]

        common.logger.message("Starting retrieving output from server "+str(self.storage_name)+"...")

        try:  
            seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
        except Exception, ex:
            common.logger.debug(1, str(ex))
            msg = "ERROR : Unable to create SE source interface \n"
            raise CrabException(msg)
        try:  
            loc = SElement("localhost", "local")
        except Exception, ex:
            common.logger.debug(1, str(ex))
            msg = "ERROR : Unable to create destination  interface \n"
            raise CrabException(msg)

        ## copy ISB ##
        sbi = SBinterface( seEl, loc )

        # retrieve them from SE #TODO replace this with SE-API
        retrievedFilesJodId = []
        for i in xrange(len(filesToRetrieve)): 
            source = osbFiles[i] 
            dest = destFiles[i]
            common.logger.debug(1, "retrieving "+ str(source) +" to "+ str(dest) )
            try:
                sbi.copy( source, dest)
            except Exception, ex:
                msg = "WARNING: Unable to retrieve output file %s \n"%osbFiles[i] 
                msg += str(ex)
                common.logger.debug(1,msg)
                continue
            ## appending jobs id related to the index "i"
            retrievedFilesJodId.append(filesToRetrieve[i])

        # notify to the server that output have been retrieved successfully. proxy from StatusServer
        if len(retrievedFilesJodId) > 0:
            common.logger.debug(5, "List of retrieved files notified to server: %s"%str(retrievedFilesJodId) ) 
            csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params)
            try:
                csCommunicator.outputRetrieved(self.taskuuid, retrievedFilesJodId)
            except Exception, e:
                pass

            common._db.updateRunJob_(retrievedFilesJodId, [{'statusScheduler':'Cleared', 'status':'E'}] * len(retrievedFilesJodId) )
        return

Revision:	1.28
Committed:	Tue May 27 15:31:28 2008 UTC (16 years, 11 months ago) by farinafa
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_2_2_pre4, CRAB_2_2_2_pre3, CRAB_2_2_2_pre2, CRAB_2_2_2_pre1, CRAB_2_2_1, CRAB_2_2_1_pre6, CRAB_2_2_1_pre5, CRAB_2_2_1_pre4, PRODCOMMON_0_10_7_testCS2
Changes since 1.27:	+4 -1 lines
Log Message:	Fix for the base64 padding mess
#	User	Rev	Content
1	spiga	1.16	from GetOutput import GetOutput
2			from StatusServer import StatusServer
3	spiga	1.1	from crab_util import *
4			import common
5			import time
6
7	farinafa	1.14	import traceback
8			from xml.dom import minidom
9			from ServerCommunicator import ServerCommunicator
10
11	spiga	1.19	from ProdCommon.Storage.SEAPI.SElement import SElement
12			from ProdCommon.Storage.SEAPI.SBinterface import SBinterface
13	farinafa	1.14
14	spiga	1.18	class GetOutputServer( GetOutput, StatusServer ):
15	farinafa	1.14
16			def __init__(self, *args):
17
18	spiga	1.21	GetOutput.__init__(self,*args)
19
20	spiga	1.22	# init client server params...
21			CliServerParams(self)
22	farinafa	1.14
23			if self.storage_path[0]!='/':
24			self.storage_path = '/'+self.storage_path
25	spiga	1.16
26	spiga	1.1	return
27	farinafa	1.14
28	spiga	1.18
29			def getOutput(self):
30	spiga	1.1
31	farinafa	1.14	# get updated status from server #inherited from StatusServer
32			self.resynchClientSide()
33	mcinquil	1.2
34	farinafa	1.14	# understand whether the required output are available
35			self.checkBeforeGet()
36	spiga	1.18
37			# retrive files
38			self.retrieveFiles(self.list_id)
39
40			self.organizeOutput()
41
42			return
43
44			def retrieveFiles(self,filesToRetrieve):
45			"""
46			Real get output from server storage
47			"""
48	spiga	1.19
49			self.taskuuid = str(common._db.queryTask('name'))
50			common.logger.debug(3, "Task name: " + self.taskuuid)
51	farinafa	1.14
52			# create the list with the actual filenames
53	spiga	1.19	remotedir = os.path.join(self.storage_path, self.taskuuid)
54	farinafa	1.14
55	spiga	1.19
56			# list of file to retrieve
57			osbTemplate = remotedir + '/out_files_%s.tgz'
58	farinafa	1.14	osbFiles = [ osbTemplate%str(jid) for jid in filesToRetrieve ]
59	spiga	1.19	common.logger.debug(3, "List of OSB files: " +str(osbFiles) )
60
61			#
62	fanzago	1.27	copyHere = self.outDir
63	spiga	1.19	destTemplate = copyHere+'/out_files_%s.tgz'
64	farinafa	1.14	destFiles = [ destTemplate%str(jid) for jid in filesToRetrieve ]
65
66	spiga	1.19	common.logger.message("Starting retrieving output from server "+str(self.storage_name)+"...")
67
68			try:
69			seEl = SElement(self.storage_name, self.storage_proto, self.storage_port)
70			except Exception, ex:
71			common.logger.debug(1, str(ex))
72			msg = "ERROR : Unable to create SE source interface \n"
73			raise CrabException(msg)
74			try:
75			loc = SElement("localhost", "local")
76			except Exception, ex:
77			common.logger.debug(1, str(ex))
78			msg = "ERROR : Unable to create destination interface \n"
79			raise CrabException(msg)
80
81			## copy ISB ##
82			sbi = SBinterface( seEl, loc )
83
84	farinafa	1.14	# retrieve them from SE #TODO replace this with SE-API
85	farinafa	1.23	retrievedFilesJodId = []
86	mcinquil	1.26	for i in xrange(len(filesToRetrieve)):
87	spiga	1.19	source = osbFiles[i]
88			dest = destFiles[i]
89			common.logger.debug(1, "retrieving "+ str(source) +" to "+ str(dest) )
90	farinafa	1.15	try:
91	spiga	1.22	sbi.copy( source, dest)
92	spiga	1.19	except Exception, ex:
93	spiga	1.20	msg = "WARNING: Unable to retrieve output file %s \n"%osbFiles[i]
94			msg += str(ex)
95			common.logger.debug(1,msg)
96	farinafa	1.15	continue
97	mcinquil	1.26	## appending jobs id related to the index "i"
98			retrievedFilesJodId.append(filesToRetrieve[i])
99	farinafa	1.14
100			# notify to the server that output have been retrieved successfully. proxy from StatusServer
101	farinafa	1.23	if len(retrievedFilesJodId) > 0:
102			common.logger.debug(5, "List of retrieved files notified to server: %s"%str(retrievedFilesJodId) )
103	spiga	1.22	csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params)
104	farinafa	1.28	try:
105			csCommunicator.outputRetrieved(self.taskuuid, retrievedFilesJodId)
106			except Exception, e:
107			pass
108	spiga	1.1
109	farinafa	1.25	common._db.updateRunJob_(retrievedFilesJodId, [{'statusScheduler':'Cleared', 'status':'E'}] * len(retrievedFilesJodId) )
110	farinafa	1.14	return
111	spiga	1.1