DATATRANSFER/loadLFN/loadLFN.py

#!/bin/env python
from ConfigParser import ConfigParser
import string 
import html as h
import os
import os.path

Usage = """
loadLFN.py 
Options:
    --file     file that does contain list of lFNs
    --out_dir  output directory 
    --verbose  verbose 
    --skip     string that has to be skipped (can have multiple values comma separeted) 
    --help     help                      
"""


class LfnType:
    def __init__(self):
       self.info_data={}
       self.info_mc={}
       self.info_relval={}
       self.info_user={}
       self.info_group={}
       self.info_results={}
       self.tmp_list=[]
       self.dqm_list=[]
       self.lumi_list=[]
       self.info_err={}

class   ConfigFile(ConfigParser):
    def __init__(self,  path=None):
        ConfigParser.__init__(self)
        self.read(path)

   
def loadData(lfn_list,lfn,cfg,values_toskip=None):

    
    data=loadList(cfg.get('DATA','data'))
    tiers=loadList(cfg.get('TIER','tiers'))
    PD_categories=loadDict(cfg.get('TIER','PD_categories'))
    data_categories=loadDict(cfg.get('TIER','data_categories'))
    msg_err={'generic':[],'type':[],'root_file':[],'file':[]}
    
    for file in lfn_list:
        
        value_toskip=False
        if values_toskip:
           for v in values_toskip:
              if file.find(v) != -1:
                 value_toskip=True
                 break
        if value_toskip:continue


        err=False    
        #removing the final \n
        file=file[:-1] 
        lst=file.split('/')
        if len(lst) <6 :
             msg_err['generic'].append("malformed file (too short %i): %s"%(len(lst),file))
             continue
        if '' in lst:lst.remove('')
        type=lst[1] 
         

        if lst[0] !="store":
           msg_err['type'].append("malformed file (doesn't contain store): %s"%(file))
           continue

        if type not in data:
           if type not in lfn.info_err.keys():
              lfn.info_err[type]={}
           err=True

        
        if lst[len(lst)-1][-4:] != 'root':
           msg_err['root_file'].append("malformed file (not a root file) :%s"%file)
  
         
        if type == 'lumi':
             lfn.lumi_list.append(file)
        elif type == 'dqm':
             lfn.dqm_list.append(file)
        elif type == 'temp':
             lfn.tmp_list.append(file)
        elif type in ('group','results'):
             groupname=lst[2]
             if type == 'group':
                if groupname not in lfn.info_group.keys():
                   lfn.info_group[groupname]=[]
                lfn.info_group[groupname].append(file)
             elif type == 'result':
                if groupname not in lfn.info_results.keys():
                   lfn.info_results[groupname]=[]
                lfn.info_results[groupname].append(file) 
        elif type == 'user':
             usrname=lst[2]
             if usrname not in lfn.info_user:
                lfn.info_user[usrname]=[]
             lfn.info_user[usrname].append(file)
        elif type in ('data', 'relval','mc'):
             data_tier=(lst[4]).strip()
             if data_tier not in tiers:
               msg_err['file'].append("malformed file (not valid data tier %s): %s" %(data_tier,file))
               continue
             
             #extract the acquisition era
             acq_era_lst=lst[2].split('-')
             if len(acq_era_lst) < 2 : acq_era_lst=lst[2].split('_')
       
             era=acq_era_lst[0]
 
             for pd in PD_categories.keys():

              if era in PD_categories[pd]:
                 era=pd
              else: 
                 for i in  PD_categories[pd]:
                     if era.startswith(i):
                        era=pd
  
              #data tier
              for dt in sorted(data_categories.keys()):
                if data_tier in data_categories[dt]:
                   data_tier=dt

             g_lfn='/%s'%'/'.join(lst[0:6])
        
             if type == 'data' :
                 
               if era not in lfn.info_data:
                  lfn.info_data[era]={}


               if data_tier not in lfn.info_data[era]:
                  lfn.info_data[era][data_tier]={}

               if g_lfn not in lfn.info_data[era][data_tier]:
                   lfn.info_data[era][data_tier][g_lfn]=[]

               lfn.info_data[era][data_tier][g_lfn].append(file)


             elif type == 'mc':
      
               if era not in lfn.info_mc:
                  lfn.info_mc[era]={}

               if data_tier not in lfn.info_mc[era]:
                  lfn.info_mc[era][data_tier]={}

               if g_lfn not in lfn.info_mc[era][data_tier]:
                   lfn.info_mc[era][data_tier][g_lfn]=[]

               lfn.info_mc[era][data_tier][g_lfn].append(file)

             elif type == 'relval':
              
               if era not in lfn.info_relval: 
                  lfn.info_relval[era]={}

               if data_tier not in lfn.info_relval[era]:
                  lfn.info_relval[era][data_tier]={}

               if g_lfn not in lfn.info_relval[era][data_tier]:
                   lfn.info_relval[era][data_tier][g_lfn]=[]

               lfn.info_relval[era][data_tier][g_lfn].append(file)

        elif  err:
          key='/%s'%'/'.join(lst[0:len(lst)-2])
          if key not in lfn.info_err[type].keys():
               lfn.info_err[type][key]=[]
          lfn.info_err[type][key].append(file)

          
    return msg_err


def printdata(cfg,lfn,dir,verbose):
    
    printMultipleDict(cfg,lfn.info_data,dir+"/data.html",'/store/data',verbose)
    printMultipleDict(cfg,lfn.info_mc,dir+"/mc.html",'/store/mc',verbose)
    printMultipleDict(cfg,lfn.info_relval,dir+"/relval.html",'/store/relval',verbose)
    printDict(lfn.info_group,dir+"/group.html",'/store/group',verbose)
    printDict(lfn.info_results,dir+"/result.html",'/store/result',verbose)
    printDict(lfn.info_user,dir+"/user.html",'/store/user',verbose)
    if len(lfn.tmp_list): printList(lfn.tmp_list,dir+"/temp.txt","/store/temp")
    if len(lfn.dqm_list): printList(lfn.dqm_list,dir+"/dqm.txt","/store/dqm")
    if len(lfn.lumi_list):printList(lfn.lumi_list,dir+"/lumi.txt","/store/lumi")
  
    dir_err=dir+"/err" 
    for i,k in lfn.info_err.iteritems():
          printDict(k,"%s/%s.html"%(dir_err,i),'/store/%s'%i,verbose)
              
    
def printList(lst,file,title):

    fd=open(file,'w')
    
    fd.write(title+'\n')
   
    for item in lst:
        
       fd.write(item+'\n') 
   
    fd.close()

def printDict(info,file,title,verbose=None):
    fd=open(file,'w')

    if len(info.keys()):
       
        fd.write( h.head)
        fd.write(h.body)
        fd.write(title)
        fd.write(h.table)
        tot_lst=[len(item) for item in info.keys()]
        
        for item in sorted(info.keys()):
            fd.write(h.tr)
            fd.write( h.td%item)
            fd.write('<td>')
            if verbose:
              fd.write( h.inner_table)
              for i in info[item]:
                fd.write(h.tr)
                fd.write('<td nowrap>%s</td>'%i)
                fd.write( h.end_tr)
              fd.write( h.end_table)
            else:
              el="%i"%len(info[item])
              fd.write(h.td%el)
            fd.write('</td>')

        fd.write(h.end_tr)
        fd.write(h.tr)
        fd.write(h.td%'TOT')
        fd.write(h.td%'&nbsp;')
        el='%i'%sum(tot_lst)
        fd.write(h.td%el)
        
        fd.write(h.end_tr)
        fd.write(h.end_table) 
        fd.write(h.end_body)      

def printMultipleDict(cfg,info,file,title,verbose=None) :

    sorted_data_categories=loadList(cfg.get('TIER','sorted_datacategories'))       
    fd=open(file,'w')

    if len(info.keys()) >0:

        fd.write( h.head)
        fd.write(h.body)
        fd.write(title)
        fd.write(h.table)
        fd.write(h.th%'&nbsp;')
        for i in sorted_data_categories: 
            fd.write( h.th%i)
        fd.write(h.th%'TOT')
    for k in info.iterkeys():
             
       item=info[k]
       tot=0
       complete_item=[item.setdefault(l,{'':[]})  for l in sorted_data_categories] 

       fd.write(h.tr)
       fd.write( h.td%k)
       for inner_dict in complete_item:
         fd.write('<td>')
         if verbose:
           fd.write( h.inner_table)
           for k,v in inner_dict.iteritems():
               fd.write(h.tr)
               el='<b>%i</b>  %s'%(len(v),k)
            #   tot=tot+len(v)
               fd.write('<td nowrap>%s</td>'%el)
               fd.write( h.end_tr)
           fd.write( h.end_table)
         else:
            fd.write('<b>%i</b>'% sum([len(v) for k,v in inner_dict.iteritems()]))
            tot=tot+ sum([len(v) for k,v in inner_dict.iteritems()])
         fd.write('</td>')

       fd.write(h.td%('%i'%tot))
       fd.write( h.end_tr)


    fd.write(h.end_table)
    fd.write(h.end_body)
 

def loadList(lst):

  if lst!=None:
    # convert it to a list
    tmp = string.split(lst,",")
    lsts = []
    for el in tmp:
      sel = string.strip(el)
      if sel!="":
         lsts.append(sel)

  return lsts


def loadDict(lst):
    dict={}
    
    if lst!=None:
        dict   = eval(lst)

    return dict

def main(opts, args):
   
    input_file=None
    verbose=False
    out_dir=None    
    values_toskip=None
 
    for opt, val in opts:
        if opt=='--help':
           print Usage
           sys.exit(1)
        if opt=='--file':
           input_file = val
        if opt=='--verbose':
           verbose=True 
        if opt=='--out_dir':
           out_dir =val
        if opt=='--skip':
           values_toskip=val.split(",")
           print values_toskip
   
    if not input_file or not out_dir: 
       print Usage
       sys.exit(1)

    if not os.path.exists(out_dir):
       print "%s doesn't exist" %out_dir
       sys.exit(1)

    if not os.path.exists(out_dir+'/err'):
        os.makedirs(out_dir+'/err')
        print "created %s/err" %out_dir

    cfg = ConfigFile( path = 'lfn.cfg')
   
    if input_file.startswith('~'): input_file= os.path.expanduser(input_file)    
    if out_dir.startswith('~'): out_dir= os.path.expanduser(out_dir)    
    
    fs=open(input_file,"r")
    lines =fs.readlines()
    fs.close()

    lfn=LfnType()
     
    msg_err=loadData(lines,lfn,cfg,values_toskip)

    for type,errLst in  msg_err.iteritems() :
       if  len(errLst):
          fd=open("%s/err/%s.err"%(out_dir,type),'w')  
          for err in errLst:
             fd.write(err+'\n')
          fd.close()    

    printdata(cfg,lfn,out_dir,verbose) 


if __name__ == '__main__':
    import sys
    import getopt
    
    try:
        opts, args = getopt.getopt(sys.argv[1:], "", ["out_dir=","file=","verbose","skip=", "help"])
    except getopt.GetoptError, msg:
        print 
        print msg
        print
        print Usage
        sys.exit(1)

    if  len(opts) <1 :
        print
        print Usage
        sys.exit(1)
        
    
    main( opts, args)
Revision:	1.2
Committed:	Wed Jul 6 19:50:01 2011 UTC (13 years, 9 months ago) by fmoscato
Content type:	text/x-python
Branch:	MAIN
Changes since 1.1:	+22 -19 lines
Log Message:	added exclusion list
#	User	Rev	Content
1	fmoscato	1.1	#!/bin/env python
2			from ConfigParser import ConfigParser
3			import string
4			import html as h
5			import os
6			import os.path
7
8			Usage = """
9			loadLFN.py
10			Options:
11			--file file that does contain list of lFNs
12			--out_dir output directory
13	fmoscato	1.2	--verbose verbose
14			--skip string that has to be skipped (can have multiple values comma separeted)
15	fmoscato	1.1	--help help
16			"""
17
18
19			class LfnType:
20			def __init__(self):
21			self.info_data={}
22			self.info_mc={}
23			self.info_relval={}
24			self.info_user={}
25			self.info_group={}
26			self.info_results={}
27			self.tmp_list=[]
28			self.dqm_list=[]
29			self.lumi_list=[]
30			self.info_err={}
31
32			class ConfigFile(ConfigParser):
33			def __init__(self, path=None):
34			ConfigParser.__init__(self)
35			self.read(path)
36
37
38
39	fmoscato	1.2	def loadData(lfn_list,lfn,cfg,values_toskip=None):
40	fmoscato	1.1
41
42			data=loadList(cfg.get('DATA','data'))
43			tiers=loadList(cfg.get('TIER','tiers'))
44			PD_categories=loadDict(cfg.get('TIER','PD_categories'))
45			data_categories=loadDict(cfg.get('TIER','data_categories'))
46			msg_err={'generic':[],'type':[],'root_file':[],'file':[]}
47
48			for file in lfn_list:
49	fmoscato	1.2
50			value_toskip=False
51			if values_toskip:
52			for v in values_toskip:
53			if file.find(v) != -1:
54			value_toskip=True
55			break
56			if value_toskip:continue
57
58
59	fmoscato	1.1
60			err=False
61			#removing the final \n
62			file=file[:-1]
63			lst=file.split('/')
64			if len(lst) <6 :
65			msg_err['generic'].append("malformed file (too short %i): %s"%(len(lst),file))
66			continue
67			if '' in lst:lst.remove('')
68			type=lst[1]
69
70
71			if lst[0] !="store":
72			msg_err['type'].append("malformed file (doesn't contain store): %s"%(file))
73			continue
74
75			if type not in data:
76			if type not in lfn.info_err.keys():
77			lfn.info_err[type]={}
78			err=True
79
80
81			if lst[len(lst)-1][-4:] != 'root':
82			msg_err['root_file'].append("malformed file (not a root file) :%s"%file)
83
84
85			if type == 'lumi':
86			lfn.lumi_list.append(file)
87			elif type == 'dqm':
88			lfn.dqm_list.append(file)
89			elif type == 'temp':
90			lfn.tmp_list.append(file)
91			elif type in ('group','results'):
92			groupname=lst[2]
93			if type == 'group':
94			if groupname not in lfn.info_group.keys():
95			lfn.info_group[groupname]=[]
96			lfn.info_group[groupname].append(file)
97			elif type == 'result':
98			if groupname not in lfn.info_results.keys():
99			lfn.info_results[groupname]=[]
100			lfn.info_results[groupname].append(file)
101			elif type == 'user':
102			usrname=lst[2]
103			if usrname not in lfn.info_user:
104			lfn.info_user[usrname]=[]
105			lfn.info_user[usrname].append(file)
106			elif type in ('data', 'relval','mc'):
107			data_tier=(lst[4]).strip()
108			if data_tier not in tiers:
109			msg_err['file'].append("malformed file (not valid data tier %s): %s" %(data_tier,file))
110			continue
111
112			#extract the acquisition era
113			acq_era_lst=lst[2].split('-')
114			if len(acq_era_lst) < 2 : acq_era_lst=lst[2].split('_')
115
116			era=acq_era_lst[0]
117
118			for pd in PD_categories.keys():
119
120			if era in PD_categories[pd]:
121			era=pd
122			else:
123			for i in PD_categories[pd]:
124			if era.startswith(i):
125			era=pd
126
127			#data tier
128			for dt in sorted(data_categories.keys()):
129			if data_tier in data_categories[dt]:
130			data_tier=dt
131
132			g_lfn='/%s'%'/'.join(lst[0:6])
133
134			if type == 'data' :
135
136			if era not in lfn.info_data:
137			lfn.info_data[era]={}
138
139
140			if data_tier not in lfn.info_data[era]:
141			lfn.info_data[era][data_tier]={}
142
143			if g_lfn not in lfn.info_data[era][data_tier]:
144			lfn.info_data[era][data_tier][g_lfn]=[]
145
146			lfn.info_data[era][data_tier][g_lfn].append(file)
147
148
149			elif type == 'mc':
150
151			if era not in lfn.info_mc:
152			lfn.info_mc[era]={}
153
154			if data_tier not in lfn.info_mc[era]:
155			lfn.info_mc[era][data_tier]={}
156
157			if g_lfn not in lfn.info_mc[era][data_tier]:
158			lfn.info_mc[era][data_tier][g_lfn]=[]
159
160			lfn.info_mc[era][data_tier][g_lfn].append(file)
161
162			elif type == 'relval':
163
164			if era not in lfn.info_relval:
165			lfn.info_relval[era]={}
166
167			if data_tier not in lfn.info_relval[era]:
168			lfn.info_relval[era][data_tier]={}
169
170			if g_lfn not in lfn.info_relval[era][data_tier]:
171			lfn.info_relval[era][data_tier][g_lfn]=[]
172
173			lfn.info_relval[era][data_tier][g_lfn].append(file)
174
175			elif err:
176			key='/%s'%'/'.join(lst[0:len(lst)-2])
177			if key not in lfn.info_err[type].keys():
178			lfn.info_err[type][key]=[]
179			lfn.info_err[type][key].append(file)
180
181
182
183			return msg_err
184
185
186			def printdata(cfg,lfn,dir,verbose):
187
188			printMultipleDict(cfg,lfn.info_data,dir+"/data.html",'/store/data',verbose)
189			printMultipleDict(cfg,lfn.info_mc,dir+"/mc.html",'/store/mc',verbose)
190	fmoscato	1.2	printMultipleDict(cfg,lfn.info_relval,dir+"/relval.html",'/store/relval',verbose)
191	fmoscato	1.1	printDict(lfn.info_group,dir+"/group.html",'/store/group',verbose)
192			printDict(lfn.info_results,dir+"/result.html",'/store/result',verbose)
193			printDict(lfn.info_user,dir+"/user.html",'/store/user',verbose)
194			if len(lfn.tmp_list): printList(lfn.tmp_list,dir+"/temp.txt","/store/temp")
195			if len(lfn.dqm_list): printList(lfn.dqm_list,dir+"/dqm.txt","/store/dqm")
196			if len(lfn.lumi_list):printList(lfn.lumi_list,dir+"/lumi.txt","/store/lumi")
197
198			dir_err=dir+"/err"
199			for i,k in lfn.info_err.iteritems():
200			printDict(k,"%s/%s.html"%(dir_err,i),'/store/%s'%i,verbose)
201
202
203
204			def printList(lst,file,title):
205
206			fd=open(file,'w')
207
208			fd.write(title+'\n')
209
210			for item in lst:
211
212			fd.write(item+'\n')
213
214			fd.close()
215
216			def printDict(info,file,title,verbose=None):
217			fd=open(file,'w')
218
219			if len(info.keys()):
220
221			fd.write( h.head)
222			fd.write(h.body)
223			fd.write(title)
224			fd.write(h.table)
225			tot_lst=[len(item) for item in info.keys()]
226
227			for item in sorted(info.keys()):
228			fd.write(h.tr)
229			fd.write( h.td%item)
230			fd.write('<td>')
231			if verbose:
232			fd.write( h.inner_table)
233			for i in info[item]:
234			fd.write(h.tr)
235	fmoscato	1.2	fd.write('<td nowrap>%s</td>'%i)
236	fmoscato	1.1	fd.write( h.end_tr)
237			fd.write( h.end_table)
238			else:
239			el="%i"%len(info[item])
240			fd.write(h.td%el)
241			fd.write('</td>')
242
243			fd.write(h.end_tr)
244			fd.write(h.tr)
245			fd.write(h.td%'TOT')
246			fd.write(h.td%' ')
247			el='%i'%sum(tot_lst)
248			fd.write(h.td%el)
249
250			fd.write(h.end_tr)
251			fd.write(h.end_table)
252			fd.write(h.end_body)
253
254			def printMultipleDict(cfg,info,file,title,verbose=None) :
255
256			sorted_data_categories=loadList(cfg.get('TIER','sorted_datacategories'))
257			fd=open(file,'w')
258
259			if len(info.keys()) >0:
260
261			fd.write( h.head)
262			fd.write(h.body)
263			fd.write(title)
264			fd.write(h.table)
265			fd.write(h.th%' ')
266			for i in sorted_data_categories:
267			fd.write( h.th%i)
268			fd.write(h.th%'TOT')
269			for k in info.iterkeys():
270
271			item=info[k]
272			tot=0
273			complete_item=[item.setdefault(l,{'':[]}) for l in sorted_data_categories]
274
275			fd.write(h.tr)
276			fd.write( h.td%k)
277			for inner_dict in complete_item:
278			fd.write('<td>')
279			if verbose:
280			fd.write( h.inner_table)
281			for k,v in inner_dict.iteritems():
282			fd.write(h.tr)
283			el='<b>%i</b> %s'%(len(v),k)
284			# tot=tot+len(v)
285	fmoscato	1.2	fd.write('<td nowrap>%s</td>'%el)
286	fmoscato	1.1	fd.write( h.end_tr)
287			fd.write( h.end_table)
288			else:
289			fd.write('<b>%i</b>'% sum([len(v) for k,v in inner_dict.iteritems()]))
290			tot=tot+ sum([len(v) for k,v in inner_dict.iteritems()])
291			fd.write('</td>')
292
293			fd.write(h.td%('%i'%tot))
294			fd.write( h.end_tr)
295
296
297
298			fd.write(h.end_table)
299			fd.write(h.end_body)
300
301
302
303			def loadList(lst):
304
305			if lst!=None:
306			# convert it to a list
307			tmp = string.split(lst,",")
308			lsts = []
309			for el in tmp:
310			sel = string.strip(el)
311			if sel!="":
312			lsts.append(sel)
313
314			return lsts
315
316
317			def loadDict(lst):
318			dict={}
319
320			if lst!=None:
321			dict = eval(lst)
322
323			return dict
324
325			def main(opts, args):
326
327			input_file=None
328			verbose=False
329			out_dir=None
330	fmoscato	1.2	values_toskip=None
331	fmoscato	1.1
332			for opt, val in opts:
333			if opt=='--help':
334			print Usage
335			sys.exit(1)
336			if opt=='--file':
337			input_file = val
338			if opt=='--verbose':
339			verbose=True
340			if opt=='--out_dir':
341			out_dir =val
342	fmoscato	1.2	if opt=='--skip':
343			values_toskip=val.split(",")
344			print values_toskip
345	fmoscato	1.1
346			if not input_file or not out_dir:
347			print Usage
348			sys.exit(1)
349
350			if not os.path.exists(out_dir):
351			print "%s doesn't exist" %out_dir
352			sys.exit(1)
353
354			if not os.path.exists(out_dir+'/err'):
355			os.makedirs(out_dir+'/err')
356			print "created %s/err" %out_dir
357
358			cfg = ConfigFile( path = 'lfn.cfg')
359
360			if input_file.startswith('~'): input_file= os.path.expanduser(input_file)
361			if out_dir.startswith('~'): out_dir= os.path.expanduser(out_dir)
362
363			fs=open(input_file,"r")
364			lines =fs.readlines()
365			fs.close()
366
367			lfn=LfnType()
368
369	fmoscato	1.2	msg_err=loadData(lines,lfn,cfg,values_toskip)
370	fmoscato	1.1
371			for type,errLst in msg_err.iteritems() :
372			if len(errLst):
373			fd=open("%s/err/%s.err"%(out_dir,type),'w')
374			for err in errLst:
375			fd.write(err+'\n')
376			fd.close()
377
378			printdata(cfg,lfn,out_dir,verbose)
379
380
381			if __name__ == '__main__':
382			import sys
383			import getopt
384
385			try:
386	fmoscato	1.2	opts, args = getopt.getopt(sys.argv[1:], "", ["out_dir=","file=","verbose","skip=", "help"])
387	fmoscato	1.1	except getopt.GetoptError, msg:
388			print
389			print msg
390			print
391			print Usage
392			sys.exit(1)
393
394			if len(opts) <1 :
395			print
396			print Usage
397			sys.exit(1)
398
399
400			main( opts, args)