CRAB/python/WorkSpace.py

from crab_exceptions import *
import logging
import common
import os, string, time
from crab_util import loadConfig

class WorkSpace:
    def __init__(self, top_dir, cfg_params):

        self._cwd_dir = os.getcwd()+'/'
        self._top_dir = top_dir    # top working directory

        self._user_out_dir = ''
        self._user_log_dir = ''
        if 'USER.outputdir' in cfg_params.keys():    
            self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
        if 'USER.logdir' in cfg_params.keys():    
            self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
        #Matteo: Necessary to manage user ui_working_dir
        if 'USER.ui_working_dir' in cfg_params.keys():    
            self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])

       # self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res')     # dir to store job results
       # self._log_dir = ("USER.logdir",self._top_dir + '/log')    # log-directory
        self._log_dir = self._top_dir + '/log'    # log-directory
        self._job_dir = self._top_dir + '/job'     # job pars, scripts, jdl's
        self._res_dir = self._top_dir + '/res'     # dir to store job results
        self._share_dir = self._top_dir + '/share' # directory for common stuff

        #self._pathForTgz = string.split(top_dir, '/')[-1]
        self._pathForTgz = self._share_dir

        self.uuid = self.uuidGen()
 
    def uuidGen(self):
        import random

        letters = "abcdefghijklmnopqrstuvwxyz"
        numbers = "0123456789"
        genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
        random.shuffle(genid_bits)
        combination = ''.join(genid_bits)

        return combination   

    def create(self):
        # Matteo change in order to ban only "data" in "CMSSW" dir and 
        # not crash when short path is given    
        subpath = self._top_dir.split('CMSSW')
        if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
            msg = 'Cannot run CRAB from "data" directory.\n'
            msg += 'please change direcotry\n'
            raise CrabException(msg)

        if not os.path.exists(self._top_dir):
            try:
                os.makedirs(self._top_dir)
            except OSError:
                msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        if not os.listdir(self._top_dir):
            os.mkdir(self._job_dir)
            os.mkdir(self._share_dir)
            os.mkdir(self._res_dir)
            os.mkdir(self._log_dir)
            pass

        # Some more check for _res_dir, since it can be user defined
        if self._user_out_dir != '':
            if not os.path.exists(self._user_out_dir):
                try: 
                    os.makedirs(self._user_out_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
                    raise CrabException(msg)
            if os.listdir(self._user_out_dir):
                msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        # ditto for _log_dir
        if self._user_log_dir != '':
            if not os.path.exists(self._user_log_dir):
                try:
                    os.makedirs(self._user_log_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
                    raise CrabException(msg)
                pass 
            if os.listdir(self._user_log_dir):
                msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        return

    def delete(self):
        """
        delete the whole workspace without doing any test!!!
        """
        common.logger.delete()
        common.logger = None

        if os.path.exists(self._top_dir):
            import shutil
            try:
                shutil.rmtree(self._top_dir)
            except :
                msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
                raise CrabException(msg)
        return

    def cwdDir(self):
        if (self._cwd_dir[-1] != '/'):
           self._cwd_dir = self._cwd_dir + '/'
        return self._cwd_dir

    def topDir(self):
        if (self._top_dir[-1] != '/'):
           self._top_dir = self._top_dir + '/'
        return self._top_dir 

    def logDir(self):
        if (self._log_dir[-1] != '/'):
           self._log_dir = self._log_dir + '/'
        return self._log_dir

    def jobDir(self):
        if (self._job_dir[-1] != '/'):
           self._job_dir = self._job_dir + '/'
        return self._job_dir

    def resDir(self):
        if (self._res_dir[-1] != '/'):
           self._res_dir = self._res_dir + '/'
        return self._res_dir

    def shareDir(self):
        if (self._share_dir[-1] != '/'):
           self._share_dir = self._share_dir + '/'
        return self._share_dir
        
    def pathForTgz(self):
        if (self._pathForTgz[-1] != '/'):
           self._pathForTgz = self._pathForTgz + '/'
        return self._pathForTgz
        
    def task_uuid(self):
        return self.uuid

    def setResDir(self, dir):
        self._res_dir = dir
        return

    def saveFileName(self):
        return self.shareDir() + common.prog_name + '.sav'

    def cfgFileName(self):
        return self.shareDir() + common.prog_name + '.cfg'

    def saveConfiguration(self, opts, pars):

        # Save options
        
        save_file = open(self.saveFileName(), 'w')

        for k in opts.keys():
            if opts[k] : save_file.write(k+'='+opts[k]+'\n')
            else       : save_file.write(k+'\n')
            pass
        
        save_file.close()

        # Save cfg-file

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass
        
        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return

    def addToSavedCfg(self, opt):
        pars={}
        pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
        pars[opt.keys()[0]]=opt.values()[0]

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass

        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return
    def loadSavedOptions(self):
        
        # Open save-file

        try:
            save_file = open(self.saveFileName(), 'r')
        except IOError, e:
            msg = 'Misconfigured continuation directory:\n'
            msg += str(e)
            raise CrabException(msg)

        # Read saved options

        save_opts = {}
        for line in save_file:
            line = line[:-1]  # drop '\n'
            try:
                (k, v) = string.split(line, '=')
            except:
                k=line
                v=''
                pass
            save_opts[k] = v
            pass
        
        save_file.close()
        return save_opts
    
Revision:	1.43
Committed:	Mon Jul 4 16:57:45 2011 UTC (13 years, 10 months ago) by spiga
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_8_0, CRAB_2_8_0_pre1, CRAB_2_7_10_pre3, CRAB_2_7_9_patch2_pre1, CRAB_2_7_10_pre2, CRAB_2_7_10_pre1, CRAB_2_7_9_patch1, CRAB_2_7_9, CRAB_2_7_9_pre5, CRAB_2_7_9_pre4, CRAB_2_7_9_pre3, CRAB_2_7_9_pre2, CRAB_2_7_9_pre1
Changes since 1.42:	+4 -0 lines
Log Message:	rollback #82422
#	User	Rev	Content
1	nsmirnov	1.1	from crab_exceptions import *
2	slacapra	1.33	import logging
3	nsmirnov	1.1	import common
4	slacapra	1.33	import os, string, time
5	spiga	1.40	from crab_util import loadConfig
6	nsmirnov	1.1
7			class WorkSpace:
8	fanzago	1.10	def __init__(self, top_dir, cfg_params):
9	farinafa	1.21
10	spiga	1.6	self._cwd_dir = os.getcwd()+'/'
11	farinafa	1.21	self._top_dir = top_dir # top working directory
12	farinafa	1.20
13	spiga	1.26	self._user_out_dir = ''
14			self._user_log_dir = ''
15			if 'USER.outputdir' in cfg_params.keys():
16			self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
17			if 'USER.logdir' in cfg_params.keys():
18			self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
19	farinafa	1.20	#Matteo: Necessary to manage user ui_working_dir
20			if 'USER.ui_working_dir' in cfg_params.keys():
21	spiga	1.25	self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])
22	farinafa	1.20
23	spiga	1.26	# self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res') # dir to store job results
24			# self._log_dir = ("USER.logdir",self._top_dir + '/log') # log-directory
25			self._log_dir = self._top_dir + '/log' # log-directory
26	nsmirnov	1.1	self._job_dir = self._top_dir + '/job' # job pars, scripts, jdl's
27	spiga	1.26	self._res_dir = self._top_dir + '/res' # dir to store job results
28	nsmirnov	1.1	self._share_dir = self._top_dir + '/share' # directory for common stuff
29	farinafa	1.20
30	slacapra	1.30	#self._pathForTgz = string.split(top_dir, '/')[-1]
31			self._pathForTgz = self._share_dir
32	farinafa	1.20
33	spiga	1.31	self.uuid = self.uuidGen()
34
35			def uuidGen(self):
36			import random
37
38			letters = "abcdefghijklmnopqrstuvwxyz"
39			numbers = "0123456789"
40			genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
41			random.shuffle(genid_bits)
42			combination = ''.join(genid_bits)
43
44			return combination
45	nsmirnov	1.1
46			def create(self):
47	farinafa	1.21	# Matteo change in order to ban only "data" in "CMSSW" dir and
48			# not crash when short path is given
49			subpath = self._top_dir.split('CMSSW')
50	spiga	1.43	if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
51			msg = 'Cannot run CRAB from "data" directory.\n'
52			msg += 'please change direcotry\n'
53			raise CrabException(msg)
54	slacapra	1.18
55	nsmirnov	1.1	if not os.path.exists(self._top_dir):
56	slacapra	1.22	try:
57	slacapra	1.38	os.makedirs(self._top_dir)
58	slacapra	1.22	except OSError:
59			msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
60			raise CrabException(msg)
61	fanzago	1.9	pass
62			if not os.listdir(self._top_dir):
63	nsmirnov	1.1	os.mkdir(self._job_dir)
64			os.mkdir(self._share_dir)
65	spiga	1.26	os.mkdir(self._res_dir)
66			os.mkdir(self._log_dir)
67	nsmirnov	1.1	pass
68	fanzago	1.10
69	slacapra	1.24	# Some more check for _res_dir, since it can be user defined
70	spiga	1.26	if self._user_out_dir != '':
71			if not os.path.exists(self._user_out_dir):
72			try:
73	slacapra	1.38	os.makedirs(self._user_out_dir)
74	spiga	1.26	except:
75			msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
76			raise CrabException(msg)
77			if os.listdir(self._user_out_dir):
78			msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
79	fanzago	1.11	raise CrabException(msg)
80	slacapra	1.24	# ditto for _log_dir
81	spiga	1.26	if self._user_log_dir != '':
82			if not os.path.exists(self._user_log_dir):
83			try:
84	slacapra	1.38	os.makedirs(self._user_log_dir)
85	spiga	1.26	except:
86			msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
87			raise CrabException(msg)
88			pass
89			if os.listdir(self._user_log_dir):
90			msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
91	fanzago	1.11	raise CrabException(msg)
92	nsmirnov	1.1	return
93
94	slacapra	1.4	def delete(self):
95			"""
96			delete the whole workspace without doing any test!!!
97			"""
98	slacapra	1.35	common.logger.delete()
99			common.logger = None
100
101	slacapra	1.4	if os.path.exists(self._top_dir):
102	slacapra	1.33	import shutil
103	farinafa	1.36	try:
104			shutil.rmtree(self._top_dir)
105			except :
106			msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
107			raise CrabException(msg)
108	slacapra	1.4	return
109
110	spiga	1.6	def cwdDir(self):
111	fanzago	1.37	if (self._cwd_dir[-1] != '/'):
112			self._cwd_dir = self._cwd_dir + '/'
113			return self._cwd_dir
114	spiga	1.6
115	nsmirnov	1.1	def topDir(self):
116	fanzago	1.37	if (self._top_dir[-1] != '/'):
117			self._top_dir = self._top_dir + '/'
118			return self._top_dir
119	nsmirnov	1.1
120			def logDir(self):
121	fanzago	1.37	if (self._log_dir[-1] != '/'):
122			self._log_dir = self._log_dir + '/'
123			return self._log_dir
124	nsmirnov	1.1
125			def jobDir(self):
126	fanzago	1.37	if (self._job_dir[-1] != '/'):
127			self._job_dir = self._job_dir + '/'
128			return self._job_dir
129	nsmirnov	1.1
130			def resDir(self):
131	fanzago	1.37	if (self._res_dir[-1] != '/'):
132			self._res_dir = self._res_dir + '/'
133			return self._res_dir
134	nsmirnov	1.1
135			def shareDir(self):
136	fanzago	1.37	if (self._share_dir[-1] != '/'):
137			self._share_dir = self._share_dir + '/'
138			return self._share_dir
139	corvo	1.14
140			def pathForTgz(self):
141	fanzago	1.37	if (self._pathForTgz[-1] != '/'):
142			self._pathForTgz = self._pathForTgz + '/'
143			return self._pathForTgz
144	corvo	1.14
145	spiga	1.32	def task_uuid(self):
146			return self.uuid
147	nsmirnov	1.1
148			def setResDir(self, dir):
149			self._res_dir = dir
150			return
151
152			def saveFileName(self):
153			return self.shareDir() + common.prog_name + '.sav'
154
155			def cfgFileName(self):
156			return self.shareDir() + common.prog_name + '.cfg'
157
158	slacapra	1.28	def saveConfiguration(self, opts, pars):
159	nsmirnov	1.1
160			# Save options
161
162			save_file = open(self.saveFileName(), 'w')
163
164			for k in opts.keys():
165	nsmirnov	1.2	if opts[k] : save_file.write(k+'='+opts[k]+'\n')
166			else : save_file.write(k+'\n')
167	nsmirnov	1.1	pass
168
169			save_file.close()
170
171			# Save cfg-file
172
173	slacapra	1.28	secs={}
174			for k in pars.keys():
175			sec, value = string.split(k,".")
176			if sec not in secs.keys():
177			secs[sec]={}
178			secs[sec][value]=pars[k]
179			pass
180
181			config_file = open(self.cfgFileName(), 'w')
182			for k in secs.keys():
183			config_file.write('['+k+']\n')
184			for v in secs[k].keys():
185	spiga	1.39	if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
186	slacapra	1.28	else : config_file.write('\n')
187			config_file.close()
188	nsmirnov	1.1
189			return
190
191	spiga	1.41	def addToSavedCfg(self, opt):
192			pars={}
193			pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
194			pars[opt.keys()[0]]=opt.values()[0]
195
196			secs={}
197			for k in pars.keys():
198			sec, value = string.split(k,".")
199			if sec not in secs.keys():
200			secs[sec]={}
201			secs[sec][value]=pars[k]
202			pass
203
204			config_file = open(self.cfgFileName(), 'w')
205			for k in secs.keys():
206			config_file.write('['+k+']\n')
207			for v in secs[k].keys():
208			if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
209			else : config_file.write('\n')
210			config_file.close()
211
212			return
213	nsmirnov	1.1	def loadSavedOptions(self):
214
215			# Open save-file
216
217			try:
218			save_file = open(self.saveFileName(), 'r')
219			except IOError, e:
220			msg = 'Misconfigured continuation directory:\n'
221			msg += str(e)
222			raise CrabException(msg)
223
224			# Read saved options
225
226			save_opts = {}
227			for line in save_file:
228			line = line[:-1] # drop '\n'
229	slacapra	1.3	try:
230			(k, v) = string.split(line, '=')
231			except:
232			k=line
233			v=''
234			pass
235	nsmirnov	1.1	save_opts[k] = v
236			pass
237
238			save_file.close()
239			return save_opts
240