| 1 |
#!/usr/bin/python
|
| 2 |
|
| 3 |
# System libraries to interact with the shell
|
| 4 |
import sys
|
| 5 |
import os
|
| 6 |
from os import popen
|
| 7 |
import commands
|
| 8 |
import time
|
| 9 |
import getpass
|
| 10 |
from sys import stdout
|
| 11 |
from optparse import OptionParser
|
| 12 |
|
| 13 |
def ensure_dir(f) :
|
| 14 |
if not os.path.exists(f):
|
| 15 |
os.makedirs(f)
|
| 16 |
|
| 17 |
|
| 18 |
def join_directory(path,filelist) :
|
| 19 |
localpath="/scratch/"+uname+"/"+path
|
| 20 |
ensure_dir(localpath)
|
| 21 |
cleanpath=path;
|
| 22 |
if (cleanpath[len(cleanpath)-1]=="/") : # remove trailing slash
|
| 23 |
cleanpath=cleanpath[0:len(cleanpath)-2]
|
| 24 |
filecounter=0
|
| 25 |
maxsizeperhadd=50
|
| 26 |
commandlist=[]
|
| 27 |
command=""
|
| 28 |
invertedcounter=len(filelist)
|
| 29 |
for item in filelist:
|
| 30 |
sys.stdout.write("\r Working on file "+str(filecounter+1)+" of "+str(len(filelist))+" ("+str(round(100*filecounter/float(len(filelist)),2))+" % done)")
|
| 31 |
sys.stdout.flush()
|
| 32 |
copycommand="lcg-cp srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN=/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+item+" file:////scratch/"+uname+"/"+item
|
| 33 |
commands.getstatusoutput(copycommand)
|
| 34 |
command=command+" "+" /scratch/"+uname+"/"+item
|
| 35 |
filecounter=filecounter+1
|
| 36 |
invertedcounter=invertedcounter-1
|
| 37 |
if filecounter == maxsizeperhadd or invertedcounter == 0:
|
| 38 |
commandlist.append(command)
|
| 39 |
command=""
|
| 40 |
filecounter=0
|
| 41 |
stdout.write("\r \r\n") #clean up
|
| 42 |
|
| 43 |
commandcounter=0
|
| 44 |
fusedfiles=[]
|
| 45 |
for command in commandlist:
|
| 46 |
commandcounter=commandcounter+1
|
| 47 |
fusefile=" /scratch/"+uname+"/"+cleanpath+"/earlystagecommand"+str(commandcounter)+".root"
|
| 48 |
fusedfiles.append(fusefile)
|
| 49 |
ccommand="hadd -f "+fusefile+" "+command
|
| 50 |
print "Fusing "+str(maxsizeperhadd)+" files"
|
| 51 |
print ccommand
|
| 52 |
commands.getoutput(ccommand)
|
| 53 |
command="hadd -f /scratch/"+uname+"/"+cleanpath+".root"
|
| 54 |
for file in fusedfiles:
|
| 55 |
command=command+" "+file
|
| 56 |
commands.getoutput(command)
|
| 57 |
|
| 58 |
# deletecommand="rm -r /scratch/"+uname+"/"+path+"/"
|
| 59 |
print commands.getoutput(deletecommand)
|
| 60 |
print "All files have been joined; the merged file is in /scratch/"+uname+"/"+cleanpath+".root"
|
| 61 |
return "/scratch/"+uname+"/"+cleanpath+".root"
|
| 62 |
|
| 63 |
|
| 64 |
def check_directory(path) :
|
| 65 |
complete_path="/pnfs/lcg.cscs.ch/cms/trivcat/store/user/"+uname+"/"+path+"/"
|
| 66 |
print "\033[1;34m Going to checkout the subdirectory "+complete_path+" \033[0m "
|
| 67 |
listoffiles=[]
|
| 68 |
supposedtobejoined=False;
|
| 69 |
commandline="lcg-ls -l srm://storage01.lcg.cscs.ch:8443/srm/managerv2?SFN="+complete_path
|
| 70 |
pipe=popen(commandline)
|
| 71 |
for l in pipe.readlines():
|
| 72 |
currentline=l.strip("\n")
|
| 73 |
if(currentline[0]=="d") :
|
| 74 |
check_directory(currentline[currentline.find(path):])
|
| 75 |
else :
|
| 76 |
if(currentline.count(path) > 0) :
|
| 77 |
supposedtobejoined=True
|
| 78 |
listoffiles.append(currentline[currentline.find(path):])
|
| 79 |
if supposedtobejoined==True:
|
| 80 |
join_directory(path,listoffiles)
|
| 81 |
|
| 82 |
def fuse_job(jobname) :
|
| 83 |
print "Going to fuse the output."
|
| 84 |
check_directory("bussolaresults/"+jobname)
|
| 85 |
|
| 86 |
def usage() :
|
| 87 |
print "Please indicate the directory you'd like to fuse, using the -d option"
|
| 88 |
sys.exit(-1)
|
| 89 |
|
| 90 |
parser = OptionParser()
|
| 91 |
parser.add_option("-d","--dir",dest="directory",help="name of directory",default=-1)
|
| 92 |
(options, args) = parser.parse_args()
|
| 93 |
|
| 94 |
jdir=options.directory
|
| 95 |
|
| 96 |
if jdir==-1:
|
| 97 |
usage()
|
| 98 |
|
| 99 |
uname=getpass.getuser()
|
| 100 |
fuse_job(str(jdir))
|
