COMP/PKGTOOLS/cmsDownloadME.py

#!/usr/bin/env python

#
# June 21, 2008
# dlange, LLNL
#
# Simple script to retrieve ME directory listings
#

import md5
import os
from os.path import join, getsize
import sys
import optparse
import urllib2

usage=\
"""%prog <TYPE> [options].
Examples:
%prog -arch slc4_ia32_gcc345 -gen madgraph --cat QCD --version 4.2.11-cms1
"""

parser=optparse.OptionParser(usage)

parser.add_option("-a","--arch",
                  help="The CMSSW architecture (default=slc4_ia32_gcc345)",
                  default="slc4_ia32_gcc345",
                  dest="arch");
parser.add_option("-g","--gen",
                  help="The generator to retrieve MEs for",
                  default="",
                  dest="gen");
parser.add_option("-c","--cat",
                  help="The category of MEs (eg, QCD)",
                  default="",
                  dest="cat");
parser.add_option("-v","--ver",
                  help="The version of MEs (eg, 4.2.11-cms1)",
                  default="",
                  dest="ver");


(options,args) = parser.parse_args() # by default the arg is sys.argv[1:]


baseDirWeb='http://cmsrep.cern.ch/cmssw/ME/'
listFileEnding='list'
cmsswbase=os.environ.get('CMS_PATH')
#hack for cern software area
if cmsswbase=='/afs/cern.ch/cms':
    cmsswbase='/afs/cern.ch/cms/sw'

if cmsswbase is None:
    print 'Missing CMSSW_BASE variable to define SW area'
    sys.exit(1)

req=urllib2.Request(baseDirWeb)
try: website=urllib2.urlopen(req)
except IOError, e:
    print 'Can not talk to cmsrep web server. Network problem?'
    sys.exit()    

dlist=website.read()

archList={}
genList={}
catList={}
versionList={}

for line in dlist.split('\n'):
    if ( line.find(listFileEnding)>-1):
        href=line.split('<a')[1].split('</a>')[0].split('">')[1]
        archList[href]=href.split(':')[0]
        genList[href]=href.split(':')[1]
        catList[href]=href.split(':')[2]
        versionList[href]=(href.split(':')[3])[0:-5]

if options.gen == '':
    print 'no generator (--gen) specified. For this arch, choices are:'
    tmpDict={}
    for ent in archList:
        if archList[ent] == options.arch:
            gen=genList[ent]
            if gen not in tmpDict:
                tmpDict[gen]=1
                print '    ' + gen
    sys.exit(0)            

if options.cat == '':
    print 'no category (--cat) specified. For this arch/generator, choices are:'
    tmpDict={}
    for ent in archList:
        if archList[ent] == options.arch:
            if genList[ent] == options.gen:
                cat=catList[ent]
                if cat not in tmpDict:
                    tmpDict[cat]=1
                    print '    ' + cat
    sys.exit(0)            

if options.ver == '':
    print 'no version (--ver) specified. For this arch/generator/category, choices are:'
    tmpDict={}
    for ent in archList:
        if archList[ent] == options.arch:
            if genList[ent] == options.gen:
                if catList[ent] == options.cat:
                    ver=versionList[ent]
                    if ver not in tmpDict:
                        tmpDict[ver]=1
                        print '    ' + ver
    sys.exit(0)            

fileExpected=options.arch+':'+options.gen+':'+options.cat+':'+options.ver+'.'+ listFileEnding

if ( fileExpected not in archList):
    print 'No MEs for arch='+options.arch+' generator='+options.gen+' category='+options.cat+' version='+options.ver+' found'
    print 'Remove --ver, --cat and/or --gen arguments to find available'
    print 'options for this architecture'
    sys.exit(0)

 # otherwise we have work to do.

 
req2=urllib2.Request(baseDirWeb+'/'+fileExpected)
try: listFile=urllib2.urlopen(req2)
except IOError, e:
    print 'Can not talk to cmsrep web server. Network problem?'
    sys.exit()    

listing=listFile.read()

for line in listing.split('\n'):
    if ( len(line.split(' '))==2):
        file=line.split(' ')[0]
        md5Server=line.split(' ')[1]
        fileOut=cmsswbase+'/'+file[file.find(baseDirWeb)+len(baseDirWeb):]
        print 'Considering: '+fileOut
        
        needToGet=0
        if ( os.path.exists(fileOut)):
            mysum=md5.md5(open(fileOut).read()).hexdigest()
            if ( mysum == md5Server):
                print '    File already downloaded'
            else:
                print '    Refetching file (changed on server)....'
                needToGet=1
        else:        
            print '    Fetching file (new)....'
            needToGet=1
            dir=os.path.dirname(fileOut)
            if not os.path.exists(dir):
                os.makedirs(dir)
            if not os.path.exists(dir):
                print 'Could not create directory to download file'
                print dir
                print 'Permissions ok?'
                sys.exit(1)

# do we need to fetch the file
        if ( needToGet==1):
        
            req3=urllib2.Request(file)
            try: listFile=urllib2.urlopen(req3)
            except IOError, e:
                print 'Can not talk to cmsrep web server. Network problem?'
                sys.exit()    
        
            fout=open(fileOut,'w')
            fout.write(listFile.read())
            fout.close()
            print '    done.'
    else:
        if ( line!=''):
            print 'Unknown line.. skipping'
            print line
Revision:	1.4
Committed:	Wed Jun 25 12:44:02 2008 UTC (16 years, 10 months ago) by dlange
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	CMSSW_3_3_3, CMSSW_3_1_5, CMSSW_3_3_2, CMSSW_3_2_8, CMSSW_3_3_0, CMSSW_3_4_0_pre1, CMSSW_3_3_0_pre6, CMSSW_3_3_0_pre5, CMSSW_3_2_7, CMSSW_3_3_0_pre4, CMSSW_3_3_0_pre3, V00-08-08, CMSSW_3_3_0_pre2, CMSSW_3_2_3, CMSSW_3_2_2_patch2, CMSSW_3_2_2_patch1, CMSSW_3_2_2, CMSSW_3_2_1, CMSSW_3_2_0, CMSSW_3_1_1_patch1, CMSSW_2_2_13_offpatch1, CMSSW_2_2_13_FWLITE, CMSSW_2_2_13_HLT, CMSSW_2_2_13, CMSSW_2_2_12_HLT, CMSSW_2_2_12, CMSSW_2_2_11_offpatch1, CMSSW_2_2_11, CMSSW_2_2_11_HLT, CMSSW_2_2_10_HLT, CMSSW_2_2_10, dl090425, CMSSW_2_2_9, CMSSW_2_2_8, CMSSW_2_2_6_HLT_ONLINE1, CMSSW_2_2_6_HLT_ONLINE, CMSSW_2_2_6_ONLINE, nr090319_for226_hlt_onl1, CMSSW_2_2_6, V00-08-07, V00-08-06, V00-08-05, V00-09-02, V00-08-04, V00-09-01, v00-08-04, apFor210, ap210, V00-09-00
Branch point for:	ap-210-branch
Changes since 1.3:	+26 -7 lines
Log Message:	still working towards first working version of generator download script
#	User	Rev	Content
1	dlange	1.1	#!/usr/bin/env python
2
3			#
4			# June 21, 2008
5			# dlange, LLNL
6			#
7			# Simple script to retrieve ME directory listings
8			#
9
10			import md5
11			import os
12			from os.path import join, getsize
13			import sys
14			import optparse
15			import urllib2
16
17			usage=\
18			"""%prog <TYPE> [options].
19			Examples:
20	dlange	1.4	%prog -arch slc4_ia32_gcc345 -gen madgraph --cat QCD --version 4.2.11-cms1
21	dlange	1.1	"""
22
23			parser=optparse.OptionParser(usage)
24
25			parser.add_option("-a","--arch",
26			help="The CMSSW architecture (default=slc4_ia32_gcc345)",
27			default="slc4_ia32_gcc345",
28			dest="arch");
29			parser.add_option("-g","--gen",
30			help="The generator to retrieve MEs for",
31			default="",
32			dest="gen");
33			parser.add_option("-c","--cat",
34			help="The category of MEs (eg, QCD)",
35			default="",
36			dest="cat");
37	dlange	1.4	parser.add_option("-v","--ver",
38			help="The version of MEs (eg, 4.2.11-cms1)",
39			default="",
40			dest="ver");
41	dlange	1.1
42
43			(options,args) = parser.parse_args() # by default the arg is sys.argv[1:]
44
45
46			baseDirWeb='http://cmsrep.cern.ch/cmssw/ME/'
47			listFileEnding='list'
48			cmsswbase=os.environ.get('CMS_PATH')
49			#hack for cern software area
50	dlange	1.2	if cmsswbase=='/afs/cern.ch/cms':
51	dlange	1.3	cmsswbase='/afs/cern.ch/cms/sw'
52	dlange	1.1
53			if cmsswbase is None:
54			print 'Missing CMSSW_BASE variable to define SW area'
55			sys.exit(1)
56
57			req=urllib2.Request(baseDirWeb)
58			try: website=urllib2.urlopen(req)
59			except IOError, e:
60			print 'Can not talk to cmsrep web server. Network problem?'
61			sys.exit()
62
63			dlist=website.read()
64
65			archList={}
66			genList={}
67			catList={}
68	dlange	1.4	versionList={}
69	dlange	1.1
70			for line in dlist.split('\n'):
71			if ( line.find(listFileEnding)>-1):
72			href=line.split('<a')[1].split('</a>')[0].split('">')[1]
73	dlange	1.4	archList[href]=href.split(':')[0]
74			genList[href]=href.split(':')[1]
75			catList[href]=href.split(':')[2]
76			versionList[href]=(href.split(':')[3])[0:-5]
77	dlange	1.1
78			if options.gen == '':
79			print 'no generator (--gen) specified. For this arch, choices are:'
80			tmpDict={}
81			for ent in archList:
82			if archList[ent] == options.arch:
83			gen=genList[ent]
84			if gen not in tmpDict:
85			tmpDict[gen]=1
86			print ' ' + gen
87			sys.exit(0)
88
89			if options.cat == '':
90			print 'no category (--cat) specified. For this arch/generator, choices are:'
91			tmpDict={}
92			for ent in archList:
93			if archList[ent] == options.arch:
94			if genList[ent] == options.gen:
95			cat=catList[ent]
96			if cat not in tmpDict:
97			tmpDict[cat]=1
98			print ' ' + cat
99			sys.exit(0)
100
101	dlange	1.4	if options.ver == '':
102			print 'no version (--ver) specified. For this arch/generator/category, choices are:'
103			tmpDict={}
104			for ent in archList:
105			if archList[ent] == options.arch:
106			if genList[ent] == options.gen:
107			if catList[ent] == options.cat:
108			ver=versionList[ent]
109			if ver not in tmpDict:
110			tmpDict[ver]=1
111			print ' ' + ver
112			sys.exit(0)
113
114			fileExpected=options.arch+':'+options.gen+':'+options.cat+':'+options.ver+'.'+ listFileEnding
115	dlange	1.1
116			if ( fileExpected not in archList):
117	dlange	1.4	print 'No MEs for arch='+options.arch+' generator='+options.gen+' category='+options.cat+' version='+options.ver+' found'
118			print 'Remove --ver, --cat and/or --gen arguments to find available'
119	dlange	1.1	print 'options for this architecture'
120			sys.exit(0)
121
122			# otherwise we have work to do.
123
124
125			req2=urllib2.Request(baseDirWeb+'/'+fileExpected)
126			try: listFile=urllib2.urlopen(req2)
127			except IOError, e:
128			print 'Can not talk to cmsrep web server. Network problem?'
129			sys.exit()
130
131			listing=listFile.read()
132
133			for line in listing.split('\n'):
134			if ( len(line.split(' '))==2):
135			file=line.split(' ')[0]
136			md5Server=line.split(' ')[1]
137			fileOut=cmsswbase+'/'+file[file.find(baseDirWeb)+len(baseDirWeb):]
138			print 'Considering: '+fileOut
139
140			needToGet=0
141			if ( os.path.exists(fileOut)):
142			mysum=md5.md5(open(fileOut).read()).hexdigest()
143			if ( mysum == md5Server):
144			print ' File already downloaded'
145			else:
146			print ' Refetching file (changed on server)....'
147			needToGet=1
148			else:
149			print ' Fetching file (new)....'
150			needToGet=1
151			dir=os.path.dirname(fileOut)
152			if not os.path.exists(dir):
153			os.makedirs(dir)
154			if not os.path.exists(dir):
155			print 'Could not create directory to download file'
156			print dir
157			print 'Permissions ok?'
158			sys.exit(1)
159
160			# do we need to fetch the file
161			if ( needToGet==1):
162
163			req3=urllib2.Request(file)
164			try: listFile=urllib2.urlopen(req3)
165			except IOError, e:
166			print 'Can not talk to cmsrep web server. Network problem?'
167			sys.exit()
168
169			fout=open(fileOut,'w')
170			fout.write(listFile.read())
171			fout.close()
172			print ' done.'
173			else:
174			if ( line!=''):
175			print 'Unknown line.. skipping'
176			print line