| Revision: | 1.5 |
| Committed: | Thu Sep 18 05:01:37 2008 UTC (16 years, 7 months ago) by dlange |
| Content type: | text/x-python |
| Branch: | MAIN |
| CVS Tags: | V00-22-02, V00-21-09, V00-22-01, V00-22-00, V00-21-08, V00-21-07, V00-21-06, V00-21-05, ge20130117, V00-21-04, V00-21-03, V00-16-21, V00-21-02, V00-21-01, V00-21-00, V00-16-20, V00-16-19, V00-20-37, V00-20-36, V00-20-35, V00-20-34, V00-20-33, V00-16-18, V00-16-17, V00-20-32, V00-16-16, V00-20-31, ge20120604-llvm, ge-V00-20-29-uploadNew, V00-16-15, V00-20-30, V00-16-14, V00-20-29, V00-20-28, V00-20-27, V00-16-05-23, V00-16-05-22, V00-19-07, V00-20-26, V00-20-25, V00-16-05-21, V00-20-24, V00-16-05-20, V00-20-22-01, V00-20-23, V00-20-22, V00-20-21, V00-16-05-19, V00-20-20, V00-20-19, V00-16-05-18, V00-20-18, V00-16-05-17, V00-20-17, V00-20-16, V00-20-15, V00-20-14, V00-16-05-16, V00-20-13, V00-16-05-15, V00-20-12, V00-20-11, V00-16-05-14, V00-20-10, V00-16-05-13, V00-16-05-12, V00-20-09, V00-20-08, V00-16-05-11, V00-16-05-10, V00-20-07, V00-20-06, V00-16-05-09, V00-20-05, V00-16-05-08, V00-19-06, V00-20-04, V00-19-05, V00-20-03, V00-19-04, V00-20-02, V00-20-01, V00-19-03, V00-20-00, V00-16-05-07, V00-19-02, V00-19-01, V00-19-00, V00-18-04, V00-16-05-06, V00-16-05-03a, V00-16-05-05, V00-18-03, V00-18-02, V00-18-01, V00-18-00, V00-17-06, V00-16-05-04, V00-17-05, V00-17-04, V00-16-05-03, CMSSW_4_3_0_pre4-ports_FWLITE, V00-16-05-02, V00-17-03, V00-16-09-03, V00-17-02, V00-17-01, V00-17-00, V00-16-09-02, V00-16-09-01, V00-16-13, V00-16-12, V00-16-11, V00-16-05-01, V00-16-10, V00-16-09, V00-11-06, V00-16-08, V00-16-07, V00-16-06, V00-16-05, V00-16-04, V00-16-03, V00-16-02, sjgFor386p1, V00-16-01, V00-16-00, V00-15-00, V00-13-03, V00-14-00, pe20100825a-for39Xrb, V00-13-02, V00-13-01, V00-13-00, CMSSW_3_6_3_onlpatch4_ONLINE, V00-12-00, V00-11-05, V00-11-04, V00-11-03, V00-11-02, ge20100212-boost, ge20100112-boost, V00-11-01, eskFor336ptch1, CMSSW_3_3_3_onlpatch5_ONLINE, CMSSW_3_3_3_onlpatch4_ONLINE, CMSSW_3_3_3_onlpatch2_ONLINE, CMSSW_3_3_3_onlpatch3_ONLINE, CMSSW_3_3_3_ONLINE, CMSSW_3_3_2_ONLINE, CMSSW_3_3_2_ONLINE-slc5, PRODAGENT_0_12_17_pre2, V00-11-00, pe20091103c-34X-slc5-gcc434, CMSSW_3_3_2_FWLITE-slc5-gcc434, CMSSW_3_3_2-slc4-gcc432, CMSSW_3_3_2-slc5-gcc434, CMSSW_3_4_0_pre3, PRODAGENT_0_12_17_pre1, PRODAGENT_0_12_16_patch2, PRODAGENT_0_12_16_patch1, sm091023b-slc5onl, CMSSW_3_3_0_onlpatch1_ONLINE, CMSSW_3_2_7_onlpatch5_ONLINE, CMSSW_3_2_7_onlpatch4_ONLINE, CMSSW_3_2_7_onlpatch3_ONLINE, CMSSW_3_2_7_onlpatch2_ONLINE, CMSSW_3_2_7_onlpatch1_ONLINE, CMSSW_3_4_0_pre1-432, PRODAGENT_0_12_16, CMSSW_3_3_0_ONLINE, CMSSW_3_2_7_ONLINE, V00-10-05, CMSSW_3_2_4_ONLINE, CMSSW_3_2_1_onlpatch1_ONLINE, CMSSW_3_2_1_ONLINE, V00-10-04, CMSSW_3_1_1_ONLINE, PRODAGENT_0_12_15_patch1, CMSSW_3_1_0_onlpatch1_ONLINE, CMSSW_3_1_0_ONLINE, CMSSW_3_1_0_pre10_ONLINE, V00-10-03, V00-10-02, V00-10-01, forPHEDEX-web-services-20090406, V00-10-00, V00-09-03, HEAD |
| Branch point for: | B00-16-09-00 |
| Changes since 1.4: | +36 -16 lines |
| Log Message: | update gen script for energy |
| # | User | Rev | Content |
|---|---|---|---|
| 1 | dlange | 1.1 | #!/usr/bin/env python |
| 2 | |||
| 3 | # | ||
| 4 | # June 21, 2008 | ||
| 5 | # dlange, LLNL | ||
| 6 | # | ||
| 7 | # Simple script to retrieve ME directory listings | ||
| 8 | # | ||
| 9 | |||
| 10 | import md5 | ||
| 11 | import os | ||
| 12 | from os.path import join, getsize | ||
| 13 | import sys | ||
| 14 | import optparse | ||
| 15 | import urllib2 | ||
| 16 | |||
| 17 | usage=\ | ||
| 18 | """%prog <TYPE> [options]. | ||
| 19 | Examples: | ||
| 20 | dlange | 1.5 | %prog -arch slc4_ia32_gcc345 -gen madgraph --cat QCD --energy 10TeV --version 4.2.11-cms1 |
| 21 | dlange | 1.1 | """ |
| 22 | |||
| 23 | parser=optparse.OptionParser(usage) | ||
| 24 | |||
| 25 | parser.add_option("-a","--arch", | ||
| 26 | help="The CMSSW architecture (default=slc4_ia32_gcc345)", | ||
| 27 | default="slc4_ia32_gcc345", | ||
| 28 | dest="arch"); | ||
| 29 | parser.add_option("-g","--gen", | ||
| 30 | help="The generator to retrieve MEs for", | ||
| 31 | default="", | ||
| 32 | dest="gen"); | ||
| 33 | parser.add_option("-c","--cat", | ||
| 34 | help="The category of MEs (eg, QCD)", | ||
| 35 | default="", | ||
| 36 | dest="cat"); | ||
| 37 | dlange | 1.5 | parser.add_option("-e","--energy", |
| 38 | help="The energy of MEs (eg, 10TeV)", | ||
| 39 | default="", | ||
| 40 | dest="energy"); | ||
| 41 | dlange | 1.4 | parser.add_option("-v","--ver", |
| 42 | help="The version of MEs (eg, 4.2.11-cms1)", | ||
| 43 | default="", | ||
| 44 | dest="ver"); | ||
| 45 | dlange | 1.1 | |
| 46 | |||
| 47 | (options,args) = parser.parse_args() # by default the arg is sys.argv[1:] | ||
| 48 | |||
| 49 | |||
| 50 | baseDirWeb='http://cmsrep.cern.ch/cmssw/ME/' | ||
| 51 | listFileEnding='list' | ||
| 52 | cmsswbase=os.environ.get('CMS_PATH') | ||
| 53 | #hack for cern software area | ||
| 54 | dlange | 1.2 | if cmsswbase=='/afs/cern.ch/cms': |
| 55 | dlange | 1.3 | cmsswbase='/afs/cern.ch/cms/sw' |
| 56 | dlange | 1.1 | |
| 57 | if cmsswbase is None: | ||
| 58 | print 'Missing CMSSW_BASE variable to define SW area' | ||
| 59 | sys.exit(1) | ||
| 60 | |||
| 61 | req=urllib2.Request(baseDirWeb) | ||
| 62 | try: website=urllib2.urlopen(req) | ||
| 63 | except IOError, e: | ||
| 64 | print 'Can not talk to cmsrep web server. Network problem?' | ||
| 65 | sys.exit() | ||
| 66 | |||
| 67 | dlist=website.read() | ||
| 68 | |||
| 69 | archList={} | ||
| 70 | genList={} | ||
| 71 | catList={} | ||
| 72 | dlange | 1.5 | energyList={} |
| 73 | dlange | 1.4 | versionList={} |
| 74 | dlange | 1.1 | |
| 75 | for line in dlist.split('\n'): | ||
| 76 | if ( line.find(listFileEnding)>-1): | ||
| 77 | href=line.split('<a')[1].split('</a>')[0].split('">')[1] | ||
| 78 | dlange | 1.4 | archList[href]=href.split(':')[0] |
| 79 | genList[href]=href.split(':')[1] | ||
| 80 | dlange | 1.5 | energyList[href]=href.split(':')[2] |
| 81 | catList[href]=href.split(':')[3] | ||
| 82 | versionList[href]=(href.split(':')[4])[0:-5] | ||
| 83 | dlange | 1.1 | |
| 84 | if options.gen == '': | ||
| 85 | print 'no generator (--gen) specified. For this arch, choices are:' | ||
| 86 | tmpDict={} | ||
| 87 | for ent in archList: | ||
| 88 | if archList[ent] == options.arch: | ||
| 89 | gen=genList[ent] | ||
| 90 | if gen not in tmpDict: | ||
| 91 | tmpDict[gen]=1 | ||
| 92 | print ' ' + gen | ||
| 93 | sys.exit(0) | ||
| 94 | |||
| 95 | dlange | 1.5 | if options.energy == '': |
| 96 | print 'no energy (--energy) specified. For this arch/generator, choices are:' | ||
| 97 | tmpDict={} | ||
| 98 | for ent in archList: | ||
| 99 | if archList[ent] == options.arch: | ||
| 100 | if genList[ent] == options.gen: | ||
| 101 | energy=energyList[ent] | ||
| 102 | if energy not in tmpDict: | ||
| 103 | tmpDict[energy]=1 | ||
| 104 | print ' ' + energy | ||
| 105 | sys.exit(0) | ||
| 106 | |||
| 107 | dlange | 1.1 | if options.cat == '': |
| 108 | dlange | 1.5 | print 'no category (--cat) specified. For this arch/generator/energy, choices are:' |
| 109 | dlange | 1.1 | tmpDict={} |
| 110 | for ent in archList: | ||
| 111 | if archList[ent] == options.arch: | ||
| 112 | if genList[ent] == options.gen: | ||
| 113 | dlange | 1.5 | if energyList[ent] == options.energy: |
| 114 | cat=catList[ent] | ||
| 115 | if cat not in tmpDict: | ||
| 116 | tmpDict[cat]=1 | ||
| 117 | print ' ' + cat | ||
| 118 | dlange | 1.1 | sys.exit(0) |
| 119 | |||
| 120 | dlange | 1.4 | if options.ver == '': |
| 121 | print 'no version (--ver) specified. For this arch/generator/category, choices are:' | ||
| 122 | tmpDict={} | ||
| 123 | for ent in archList: | ||
| 124 | if archList[ent] == options.arch: | ||
| 125 | if genList[ent] == options.gen: | ||
| 126 | dlange | 1.5 | if energyList[ent] == options.energy: |
| 127 | if catList[ent] == options.cat: | ||
| 128 | ver=versionList[ent] | ||
| 129 | if ver not in tmpDict: | ||
| 130 | tmpDict[ver]=1 | ||
| 131 | print ' ' + ver | ||
| 132 | dlange | 1.4 | sys.exit(0) |
| 133 | |||
| 134 | dlange | 1.5 | fileExpected=options.arch+':'+options.gen+':'+options.energy+':'+options.cat+':'+options.ver+'.'+ listFileEnding |
| 135 | dlange | 1.1 | |
| 136 | if ( fileExpected not in archList): | ||
| 137 | dlange | 1.5 | print 'No MEs for arch='+options.arch+' generator='+options.gen+' energy='+options.energy+' category='+options.cat+' version='+options.ver+' found' |
| 138 | print 'Remove --ver, --cat, --energy and/or --gen arguments to find available' | ||
| 139 | dlange | 1.1 | print 'options for this architecture' |
| 140 | sys.exit(0) | ||
| 141 | |||
| 142 | # otherwise we have work to do. | ||
| 143 | |||
| 144 | |||
| 145 | req2=urllib2.Request(baseDirWeb+'/'+fileExpected) | ||
| 146 | try: listFile=urllib2.urlopen(req2) | ||
| 147 | except IOError, e: | ||
| 148 | print 'Can not talk to cmsrep web server. Network problem?' | ||
| 149 | sys.exit() | ||
| 150 | |||
| 151 | listing=listFile.read() | ||
| 152 | |||
| 153 | for line in listing.split('\n'): | ||
| 154 | if ( len(line.split(' '))==2): | ||
| 155 | file=line.split(' ')[0] | ||
| 156 | md5Server=line.split(' ')[1] | ||
| 157 | fileOut=cmsswbase+'/'+file[file.find(baseDirWeb)+len(baseDirWeb):] | ||
| 158 | print 'Considering: '+fileOut | ||
| 159 | |||
| 160 | needToGet=0 | ||
| 161 | if ( os.path.exists(fileOut)): | ||
| 162 | mysum=md5.md5(open(fileOut).read()).hexdigest() | ||
| 163 | if ( mysum == md5Server): | ||
| 164 | print ' File already downloaded' | ||
| 165 | else: | ||
| 166 | print ' Refetching file (changed on server)....' | ||
| 167 | needToGet=1 | ||
| 168 | else: | ||
| 169 | print ' Fetching file (new)....' | ||
| 170 | needToGet=1 | ||
| 171 | dir=os.path.dirname(fileOut) | ||
| 172 | if not os.path.exists(dir): | ||
| 173 | os.makedirs(dir) | ||
| 174 | if not os.path.exists(dir): | ||
| 175 | print 'Could not create directory to download file' | ||
| 176 | print dir | ||
| 177 | print 'Permissions ok?' | ||
| 178 | sys.exit(1) | ||
| 179 | |||
| 180 | # do we need to fetch the file | ||
| 181 | if ( needToGet==1): | ||
| 182 | |||
| 183 | req3=urllib2.Request(file) | ||
| 184 | try: listFile=urllib2.urlopen(req3) | ||
| 185 | except IOError, e: | ||
| 186 | print 'Can not talk to cmsrep web server. Network problem?' | ||
| 187 | sys.exit() | ||
| 188 | |||
| 189 | fout=open(fileOut,'w') | ||
| 190 | fout.write(listFile.read()) | ||
| 191 | fout.close() | ||
| 192 | print ' done.' | ||
| 193 | else: | ||
| 194 | if ( line!=''): | ||
| 195 | print 'Unknown line.. skipping' | ||
| 196 | print line |