| 1 |
buchmann |
1.1 |
#!/usr/bin/python
|
| 2 |
|
|
|
| 3 |
|
|
# System libraries to interact with the shell
|
| 4 |
|
|
import sys
|
| 5 |
|
|
import os
|
| 6 |
|
|
from os import popen
|
| 7 |
|
|
import commands
|
| 8 |
|
|
import time
|
| 9 |
|
|
import getpass
|
| 10 |
|
|
from sys import stdout
|
| 11 |
|
|
from optparse import OptionParser
|
| 12 |
|
|
|
| 13 |
|
|
def ensure_dir(f) :
|
| 14 |
|
|
if not os.path.exists(f):
|
| 15 |
|
|
os.makedirs(f)
|
| 16 |
|
|
|
| 17 |
|
|
|
| 18 |
|
|
def join_directory(path,filelist) :
|
| 19 |
|
|
localpath="/scratch/"+uname+"/"+path
|
| 20 |
|
|
ensure_dir(localpath)
|
| 21 |
|
|
cleanpath=path;
|
| 22 |
|
|
if (cleanpath[len(cleanpath)-1]=="/") : # remove trailing slash
|
| 23 |
|
|
cleanpath=cleanpath[0:len(cleanpath)-2]
|
| 24 |
|
|
filecounter=0
|
| 25 |
|
|
maxsizeperhadd=50
|
| 26 |
|
|
commandlist=[]
|
| 27 |
|
|
command=""
|
| 28 |
|
|
invertedcounter=len(filelist)
|
| 29 |
|
|
for item in filelist:
|
| 30 |
|
|
sys.stdout.write("\r Working on file "+str(filecounter+1)+" of "+str(len(filelist))+" ("+str(round(100*filecounter/float(len(filelist)),2))+" % done)")
|
| 31 |
|
|
sys.stdout.flush()
|
| 32 |
|
|
copycommand="lcg-cp srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN=/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+item+" file:////scratch/"+uname+"/"+item
|
| 33 |
|
|
commands.getstatusoutput(copycommand)
|
| 34 |
|
|
command=command+" "+" /scratch/"+uname+"/"+item
|
| 35 |
|
|
filecounter=filecounter+1
|
| 36 |
|
|
invertedcounter=invertedcounter-1
|
| 37 |
|
|
if filecounter == maxsizeperhadd or invertedcounter == 0:
|
| 38 |
|
|
commandlist.append(command)
|
| 39 |
|
|
command=""
|
| 40 |
|
|
filecounter=0
|
| 41 |
|
|
stdout.write("\r \r\n") #clean up
|
| 42 |
|
|
|
| 43 |
|
|
commandcounter=0
|
| 44 |
|
|
fusedfiles=[]
|
| 45 |
|
|
for command in commandlist:
|
| 46 |
|
|
commandcounter=commandcounter+1
|
| 47 |
|
|
fusefile=" /scratch/"+uname+"/"+cleanpath+"/earlystagecommand"+str(commandcounter)+".root"
|
| 48 |
|
|
fusedfiles.append(fusefile)
|
| 49 |
|
|
ccommand="hadd -f "+fusefile+" "+command
|
| 50 |
|
|
print "Fusing "+str(maxsizeperhadd)+" files"
|
| 51 |
|
|
print ccommand
|
| 52 |
|
|
commands.getoutput(ccommand)
|
| 53 |
|
|
command="hadd -f /scratch/"+uname+"/"+cleanpath+".root"
|
| 54 |
|
|
for file in fusedfiles:
|
| 55 |
|
|
command=command+" "+file
|
| 56 |
|
|
commands.getoutput(command)
|
| 57 |
|
|
|
| 58 |
|
|
# deletecommand="rm -r /scratch/"+uname+"/"+path+"/"
|
| 59 |
|
|
print commands.getoutput(deletecommand)
|
| 60 |
|
|
print "All files have been joined; the merged file is in /scratch/"+uname+"/"+cleanpath+".root"
|
| 61 |
|
|
return "/scratch/"+uname+"/"+cleanpath+".root"
|
| 62 |
|
|
|
| 63 |
|
|
|
| 64 |
|
|
def check_directory(path) :
|
| 65 |
|
|
complete_path="/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+path+"/"
|
| 66 |
|
|
print "\033[1;34m Going to checkout the subdirectory "+complete_path+" \033[0m "
|
| 67 |
|
|
listoffiles=[]
|
| 68 |
|
|
supposedtobejoined=False;
|
| 69 |
|
|
commandline="lcg-ls -l srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN="+complete_path
|
| 70 |
|
|
pipe=popen(commandline)
|
| 71 |
|
|
for l in pipe.readlines():
|
| 72 |
|
|
currentline=l.strip("\n")
|
| 73 |
|
|
if(currentline[0]=="d") :
|
| 74 |
|
|
check_directory(currentline[currentline.find(path):])
|
| 75 |
|
|
else :
|
| 76 |
|
|
if(currentline.count(path) > 0) :
|
| 77 |
|
|
supposedtobejoined=True
|
| 78 |
|
|
listoffiles.append(currentline[currentline.find(path):])
|
| 79 |
|
|
if supposedtobejoined==True:
|
| 80 |
|
|
join_directory(path,listoffiles)
|
| 81 |
|
|
|
| 82 |
|
|
def fuse_job(jobname) :
|
| 83 |
|
|
print "Going to fuse the output."
|
| 84 |
|
|
check_directory("bussolaresults/"+jobname)
|
| 85 |
|
|
|
| 86 |
|
|
def usage() :
|
| 87 |
|
|
print "Please indicate the directory you'd like to fuse, using the -d option"
|
| 88 |
|
|
sys.exit(-1)
|
| 89 |
|
|
|
| 90 |
|
|
parser = OptionParser()
|
| 91 |
|
|
parser.add_option("-d","--dir",dest="directory",help="name of directory",default=-1)
|
| 92 |
|
|
(options, args) = parser.parse_args()
|
| 93 |
|
|
|
| 94 |
|
|
jdir=options.directory
|
| 95 |
|
|
|
| 96 |
|
|
if jdir==-1:
|
| 97 |
|
|
usage()
|
| 98 |
|
|
|
| 99 |
|
|
uname=getpass.getuser()
|
| 100 |
|
|
fuse_job(str(jdir))
|
