CRAB/python/WorkSpace.py

from crab_exceptions import *
import logging
import common
import os, string, time
from crab_util import loadConfig

class WorkSpace:
    def __init__(self, top_dir, cfg_params):

        self._cwd_dir = os.getcwd()+'/'
        self._top_dir = top_dir    # top working directory

        self._user_out_dir = ''
        self._user_log_dir = ''
        if 'USER.outputdir' in cfg_params.keys():    
            self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
            if cfg_params['CRAB.scheduler'].upper() == 'CAF' :
                msg = "Cannot use 'outputdir' parameter with CAF scheduler\n"
                msg += 'please remove it from your crab.cfg\n'
                raise CrabException(msg)
        if 'USER.logdir' in cfg_params.keys():    
            self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
        #Matteo: Necessary to manage user ui_working_dir
        if 'USER.ui_working_dir' in cfg_params.keys():    
            self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])

       # self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res')     # dir to store job results
       # self._log_dir = ("USER.logdir",self._top_dir + '/log')    # log-directory
        self._log_dir = self._top_dir + '/log'    # log-directory
        self._job_dir = self._top_dir + '/job'     # job pars, scripts, jdl's
        self._res_dir = self._top_dir + '/res'     # dir to store job results
        self._share_dir = self._top_dir + '/share' # directory for common stuff

        #self._pathForTgz = string.split(top_dir, '/')[-1]
        self._pathForTgz = self._share_dir

        self.uuid = self.uuidGen()
 
    def uuidGen(self):
        import random

        letters = "abcdefghijklmnopqrstuvwxyz"
        numbers = "0123456789"
        genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
        random.shuffle(genid_bits)
        combination = ''.join(genid_bits)

        return combination   

    def create(self):
        # Matteo change in order to ban only "data" in "CMSSW" dir and 
        # not crash when short path is given    
        subpath = self._top_dir.split('CMSSW')
        if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
            msg = 'Cannot run CRAB from "data" directory.\n'
            msg += 'please change direcotry\n'
            raise CrabException(msg)

        if not os.path.exists(self._top_dir):
            try:
                os.makedirs(self._top_dir)
            except OSError:
                msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
                raise CrabException(msg)
            pass
        if not os.listdir(self._top_dir):
            os.mkdir(self._job_dir)
            os.mkdir(self._share_dir)
            os.mkdir(self._res_dir)
            os.mkdir(self._log_dir)
            pass

        # Some more check for _res_dir, since it can be user defined
        if self._user_out_dir != '':
            if not os.path.exists(self._user_out_dir):
                try: 
                    os.makedirs(self._user_out_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
                    raise CrabException(msg)
            if os.listdir(self._user_out_dir):
                msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        # ditto for _log_dir
        if self._user_log_dir != '':
            if not os.path.exists(self._user_log_dir):
                try:
                    os.makedirs(self._user_log_dir)
                except:
                    msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
                    raise CrabException(msg)
                pass 
            if os.listdir(self._user_log_dir):
                msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
                raise CrabException(msg)
        return

    def delete(self):
        """
        delete the whole workspace without doing any test!!!
        """
        common.logger.delete()
        common.logger = None

        if os.path.exists(self._top_dir):
            import shutil
            try:
                shutil.rmtree(self._top_dir)
            except :
                msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
                raise CrabException(msg)
        return

    def cwdDir(self):
        if (self._cwd_dir[-1] != '/'):
           self._cwd_dir = self._cwd_dir + '/'
        return self._cwd_dir

    def topDir(self):
        if (self._top_dir[-1] != '/'):
           self._top_dir = self._top_dir + '/'
        return self._top_dir 

    def logDir(self):
        if (self._log_dir[-1] != '/'):
           self._log_dir = self._log_dir + '/'
        return self._log_dir

    def jobDir(self):
        if (self._job_dir[-1] != '/'):
           self._job_dir = self._job_dir + '/'
        return self._job_dir

    def resDir(self):
        if (self._res_dir[-1] != '/'):
           self._res_dir = self._res_dir + '/'
        return self._res_dir

    def shareDir(self):
        if (self._share_dir[-1] != '/'):
           self._share_dir = self._share_dir + '/'
        return self._share_dir
        
    def pathForTgz(self):
        if (self._pathForTgz[-1] != '/'):
           self._pathForTgz = self._pathForTgz + '/'
        return self._pathForTgz
        
    def task_uuid(self):
        return self.uuid

    def setResDir(self, dir):
        self._res_dir = dir
        return

    def saveFileName(self):
        return self.shareDir() + common.prog_name + '.sav'

    def cfgFileName(self):
        return self.shareDir() + common.prog_name + '.cfg'

    def saveConfiguration(self, opts, pars):

        # Save options
        
        save_file = open(self.saveFileName(), 'w')

        for k in opts.keys():
            if opts[k] : save_file.write(k+'='+opts[k]+'\n')
            else       : save_file.write(k+'\n')
            pass
        
        save_file.close()

        # Save cfg-file

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass
        
        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return

    def addToSavedCfg(self, opt):
        pars={}
        pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
        pars[opt.keys()[0]]=opt.values()[0]

        secs={}
        for k in pars.keys():
            sec, value = string.split(k,".")
            if sec not in secs.keys():
                secs[sec]={}
            secs[sec][value]=pars[k]
            pass

        config_file = open(self.cfgFileName(), 'w')
        for k in secs.keys():
            config_file.write('['+k+']\n')
            for v in secs[k].keys():
                if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
            else : config_file.write('\n')
        config_file.close()

        return
    def loadSavedOptions(self):
        
        # Open save-file

        try:
            save_file = open(self.saveFileName(), 'r')
        except IOError, e:
            msg = 'Misconfigured continuation directory:\n'
            msg += str(e)
            raise CrabException(msg)

        # Read saved options

        save_opts = {}
        for line in save_file:
            line = line[:-1]  # drop '\n'
            try:
                (k, v) = string.split(line, '=')
            except:
                k=line
                v=''
                pass
            save_opts[k] = v
            pass
        
        save_file.close()
        return save_opts
    
Revision:	1.44
Committed:	Fri Mar 16 15:02:26 2012 UTC (13 years, 1 month ago) by belforte
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CRAB_2_9_1, CRAB_2_9_1_pre2, CRAB_2_9_1_pre1, CRAB_2_9_0, CRAB_2_9_0_pre2, CRAB_2_9_0_pre1, CRAB_2_8_8, CRAB_2_8_8_pre1, CRAB_2_8_7_patch3, CRAB_2_8_7_patch2, CRAB_2_8_7_patch1, CRAB_2_8_7, CRAB_2_8_7_pre2, CRAB_2_8_7_pre1, CRAB_2_8_6, CRAB_2_8_6_pre1, CRAB_2_8_5_patch3, CRAB_2_8_5_patch2, CRAB_2_8_5_patch1, CRAB_2_8_5, CRAB_2_8_5_pre5, CRAB_2_8_5_pre4, CRAB_2_8_5_pre3, CRAB_2_8_4_patch3, CRAB_2_8_5_pre2, CRAB_2_8_4_patch2, CRAB_2_8_5_pre1, CRAB_2_8_4_patch1, CRAB_2_8_4, CRAB_2_8_4_pre5, CRAB_2_8_4_pre4, CRAB_2_8_4_pre3, CRAB_2_8_4_pre2, CRAB_2_8_4_pre1, CRAB_2_8_3, CRAB_2_8_3_pre4, CRAB_2_8_3_pre3, CRAB_2_8_3_pre2, CRAB_2_8_3_pre1, CRAB_2_8_2_patch1, CRAB_2_8_2, CRAB_2_8_2_pre5, CRAB_2_8_2_pre4, CRAB_2_8_2_pre3, CRAB_2_8_2_pre2, CRAB_2_8_2_pre1, CRAB_2_8_1, HEAD
Changes since 1.43:	+4 -0 lines
Log Message:	do not allow outputdir for CAF https://savannah.cern.ch/bugs/index.php?88510
#	Content
1	from crab_exceptions import *
2	import logging
3	import common
4	import os, string, time
5	from crab_util import loadConfig
6
7	class WorkSpace:
8	def __init__(self, top_dir, cfg_params):
9
10	self._cwd_dir = os.getcwd()+'/'
11	self._top_dir = top_dir # top working directory
12
13	self._user_out_dir = ''
14	self._user_log_dir = ''
15	if 'USER.outputdir' in cfg_params.keys():
16	self._user_out_dir = os.path.abspath(cfg_params['USER.outputdir'])
17	if cfg_params['CRAB.scheduler'].upper() == 'CAF' :
18	msg = "Cannot use 'outputdir' parameter with CAF scheduler\n"
19	msg += 'please remove it from your crab.cfg\n'
20	raise CrabException(msg)
21	if 'USER.logdir' in cfg_params.keys():
22	self._user_log_dir = os.path.abspath(cfg_params['USER.logdir'])
23	#Matteo: Necessary to manage user ui_working_dir
24	if 'USER.ui_working_dir' in cfg_params.keys():
25	self._top_dir = os.path.abspath(cfg_params['USER.ui_working_dir'])
26
27	# self._res_dir = cfg_params.get("USER.outputdir", self._top_dir + '/res') # dir to store job results
28	# self._log_dir = ("USER.logdir",self._top_dir + '/log') # log-directory
29	self._log_dir = self._top_dir + '/log' # log-directory
30	self._job_dir = self._top_dir + '/job' # job pars, scripts, jdl's
31	self._res_dir = self._top_dir + '/res' # dir to store job results
32	self._share_dir = self._top_dir + '/share' # directory for common stuff
33
34	#self._pathForTgz = string.split(top_dir, '/')[-1]
35	self._pathForTgz = self._share_dir
36
37	self.uuid = self.uuidGen()
38
39	def uuidGen(self):
40	import random
41
42	letters = "abcdefghijklmnopqrstuvwxyz"
43	numbers = "0123456789"
44	genid_bits = random.sample(letters, 3) + random.sample(numbers, 3)
45	random.shuffle(genid_bits)
46	combination = ''.join(genid_bits)
47
48	return combination
49
50	def create(self):
51	# Matteo change in order to ban only "data" in "CMSSW" dir and
52	# not crash when short path is given
53	subpath = self._top_dir.split('CMSSW')
54	if len(subpath)!=1 and ("data" in subpath[-1].split("/")):
55	msg = 'Cannot run CRAB from "data" directory.\n'
56	msg += 'please change direcotry\n'
57	raise CrabException(msg)
58
59	if not os.path.exists(self._top_dir):
60	try:
61	os.makedirs(self._top_dir)
62	except OSError:
63	msg = 'Cannot create '+str(self._top_dir) +' directory.\n'
64	raise CrabException(msg)
65	pass
66	if not os.listdir(self._top_dir):
67	os.mkdir(self._job_dir)
68	os.mkdir(self._share_dir)
69	os.mkdir(self._res_dir)
70	os.mkdir(self._log_dir)
71	pass
72
73	# Some more check for _res_dir, since it can be user defined
74	if self._user_out_dir != '':
75	if not os.path.exists(self._user_out_dir):
76	try:
77	os.makedirs(self._user_out_dir)
78	except:
79	msg = 'Cannot mkdir ' + self._user_out_dir + ' Check permission'
80	raise CrabException(msg)
81	if os.listdir(self._user_out_dir):
82	msg = self._user_out_dir + ' already exists and is not empty. Please remove it before create new task'
83	raise CrabException(msg)
84	# ditto for _log_dir
85	if self._user_log_dir != '':
86	if not os.path.exists(self._user_log_dir):
87	try:
88	os.makedirs(self._user_log_dir)
89	except:
90	msg = 'Cannot mkdir ' + self._user_log_dir + ' Check permission'
91	raise CrabException(msg)
92	pass
93	if os.listdir(self._user_log_dir):
94	msg = self._user_log_dir + ' already exists and is not empty. Please remove it before create new task'
95	raise CrabException(msg)
96	return
97
98	def delete(self):
99	"""
100	delete the whole workspace without doing any test!!!
101	"""
102	common.logger.delete()
103	common.logger = None
104
105	if os.path.exists(self._top_dir):
106	import shutil
107	try:
108	shutil.rmtree(self._top_dir)
109	except :
110	msg = 'Problem removing whole workspace. Please remove the remaining files manually.'
111	raise CrabException(msg)
112	return
113
114	def cwdDir(self):
115	if (self._cwd_dir[-1] != '/'):
116	self._cwd_dir = self._cwd_dir + '/'
117	return self._cwd_dir
118
119	def topDir(self):
120	if (self._top_dir[-1] != '/'):
121	self._top_dir = self._top_dir + '/'
122	return self._top_dir
123
124	def logDir(self):
125	if (self._log_dir[-1] != '/'):
126	self._log_dir = self._log_dir + '/'
127	return self._log_dir
128
129	def jobDir(self):
130	if (self._job_dir[-1] != '/'):
131	self._job_dir = self._job_dir + '/'
132	return self._job_dir
133
134	def resDir(self):
135	if (self._res_dir[-1] != '/'):
136	self._res_dir = self._res_dir + '/'
137	return self._res_dir
138
139	def shareDir(self):
140	if (self._share_dir[-1] != '/'):
141	self._share_dir = self._share_dir + '/'
142	return self._share_dir
143
144	def pathForTgz(self):
145	if (self._pathForTgz[-1] != '/'):
146	self._pathForTgz = self._pathForTgz + '/'
147	return self._pathForTgz
148
149	def task_uuid(self):
150	return self.uuid
151
152	def setResDir(self, dir):
153	self._res_dir = dir
154	return
155
156	def saveFileName(self):
157	return self.shareDir() + common.prog_name + '.sav'
158
159	def cfgFileName(self):
160	return self.shareDir() + common.prog_name + '.cfg'
161
162	def saveConfiguration(self, opts, pars):
163
164	# Save options
165
166	save_file = open(self.saveFileName(), 'w')
167
168	for k in opts.keys():
169	if opts[k] : save_file.write(k+'='+opts[k]+'\n')
170	else : save_file.write(k+'\n')
171	pass
172
173	save_file.close()
174
175	# Save cfg-file
176
177	secs={}
178	for k in pars.keys():
179	sec, value = string.split(k,".")
180	if sec not in secs.keys():
181	secs[sec]={}
182	secs[sec][value]=pars[k]
183	pass
184
185	config_file = open(self.cfgFileName(), 'w')
186	for k in secs.keys():
187	config_file.write('['+k+']\n')
188	for v in secs[k].keys():
189	if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
190	else : config_file.write('\n')
191	config_file.close()
192
193	return
194
195	def addToSavedCfg(self, opt):
196	pars={}
197	pars = loadConfig(self.shareDir() + common.prog_name + '.cfg', pars)
198	pars[opt.keys()[0]]=opt.values()[0]
199
200	secs={}
201	for k in pars.keys():
202	sec, value = string.split(k,".")
203	if sec not in secs.keys():
204	secs[sec]={}
205	secs[sec][value]=pars[k]
206	pass
207
208	config_file = open(self.cfgFileName(), 'w')
209	for k in secs.keys():
210	config_file.write('['+k+']\n')
211	for v in secs[k].keys():
212	if v!='debug': config_file.write(v+'='+secs[k][v]+'\n')
213	else : config_file.write('\n')
214	config_file.close()
215
216	return
217	def loadSavedOptions(self):
218
219	# Open save-file
220
221	try:
222	save_file = open(self.saveFileName(), 'r')
223	except IOError, e:
224	msg = 'Misconfigured continuation directory:\n'
225	msg += str(e)
226	raise CrabException(msg)
227
228	# Read saved options
229
230	save_opts = {}
231	for line in save_file:
232	line = line[:-1] # drop '\n'
233	try:
234	(k, v) = string.split(line, '=')
235	except:
236	k=line
237	v=''
238	pass
239	save_opts[k] = v
240	pass
241
242	save_file.close()
243	return save_opts
244