MitAna/bin/mergeHist.py

#!/usr/bin/env python
#===================================================================================================
import sys, getopt, os, fnmatch, commands
import string

# declare
setupCmsswCommand = \
  'cd /home/$USER/cms/cmssw/017/CMSSW_3_9_5_patch1/src;eval `scram runtime -sh`;cd - >& /dev/null;'

#===================================================================================================
def usage():
    print "possible options are: --help, --InputPath=<myInputPath>, --OutputPath=<myOutputPath>," \
          + " --FilenameHeader=<myFileHeader>, --DatasetListFile=<datasetListFile>"

#===================================================================================================
def filesExist(path,filenameExpression):
    exists = False
    
    if not os.path.exists(path):
      return exists;
    
    for fileName in os.listdir(path):
        if fnmatch.fnmatch (fileName,filenameExpression):
            exists = True
    return exists

#===================================================================================================
# Merge Filesets
#===================================================================================================
def MergeFilesets(versionList,datasetNameList,skimNameList,inputPath,outputPath,filenameHeader):
    n = 0
    for dataset in datasetNameList:
        #print "================================================================"
        print " Merging files for dataset: " + dataset

        # create the output directory in case it is not yet there
        command              = 'mkdir -p ' + outputPath
        os.system(command)

        outputMergedFilename = filenameHeader + '_' + dataset + '_' + skimNameList[n] + '.root'
        inputFilenameRegExp  = filenameHeader + '_' + dataset + '_' + skimNameList[n] + '_????.root'
#        command              = setupCmsswCommand + 'hadd -f ' + outputPath + outputMergedFilename \
        command              = 'hadd -f7 ' + outputPath + outputMergedFilename \
                               + ' ' + inputPath + '/' + versionList[n] + '/' + dataset + '/' \
                               + inputFilenameRegExp + ' >& ./merging.tmp'
#        command              = 'hadd -f ' + outputPath + \
#                               outputMergedFilename + ' ' + inputPath + inputFilenameRegExp + \
#                               ' >& ./merging.tmp'

        if (filesExist(inputPath+'/'+versionList[n]+'/'+dataset,inputFilenameRegExp) == True):
            if (os.path.exists(outputPath+outputMergedFilename)):
                print " Warning: merged file already exists. It will be deleted.\n " + \
                      outputPath+outputMergedFilename
                os.system('rm ' + outputPath+outputMergedFilename)
            #print ' merging: ' + command
            os.system(command)
            #print ''
        else:
            print " Warning: No files for dataset " + dataset + "\n at the location: " + inputPath \
                  + '/' + versionList[n] + '/' + dataset + '/' + inputFilenameRegExp
        #print ''
        n += 1

#===================================================================================================
# Main Program
#===================================================================================================
datasetListFile = ''
inputPath = ''
outputPath = ''
filenameHeader = ''
versionList = list()
datasetNameList = list()
skimNameList = list()

if len(sys.argv[1:]) < 1:
    print "Error: not enough parameters specified"
    usage()
    sys.exit()

try:
    opts, args = getopt.getopt(sys.argv[1:], "hi:o:f:d:",
                               ["help","InputPath=","OutputPath=",
                                "FilenameHeader=","DatasetListFile="])
    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-i", "--InputPath"):
            inputPath = a + "/"
        elif o in ("-o", "--OutputPath"):
            outputPath = a + "/"
        elif o in ("-f", "--FilenameHeader"):
            filenameHeader = a
        elif o in ("-d", "--DatasetListFile"):
            datasetListFile = a
        else:
            usage()
            sys.exit()
except getopt.GetoptError:
    usage()
    sys.exit(2)

if (inputPath == ''):
    print "Error: No InputPath specified."
    sys.exit()

if (outputPath == ''):
    print "Error: No OutputPath specified."
    sys.exit()

if (filenameHeader == ''):
    print "Error: No FilenameHeader specified."
    sys.exit()

if (datasetListFile == ''):
    print "Error: No dataset list file specified."
    sys.exit()

try:
    inputFile = open(datasetListFile,"r")
except IOError:
    print "Error: The specified dataset list file " + datasetListFile + " could not be opened."
    sys.exit()

print datasetListFile
#===================================================================================================
# Read in list of datasets and skim names
#===================================================================================================
lineNumber = 1
templine = inputFile.readline()
while len(templine.split()) > 0:

   # print "reading line"
   # print templine

    # ignore commented lines
    if (templine[0] == '#'):
        templine = inputFile.readline()
        lineNumber += 1
        continue;


   # print "processing line"
    # check what type of list was provided and assume 'noskim' as default
    if (len(templine.split()) == 7) :
        tempInputList = templine.split()
        versionList    .append(tempInputList[0])
        datasetNameList.append(tempInputList[1])
        skimNameList   .append('noskim')
        lineNumber += 1
    else:
        print " ERROR: incorrect format for cross section file. Check line %s" % lineNumber
        sys.exit()

    # read the next line
    templine = inputFile.readline()

inputFile.close()

# Check the list of variables
count = 0
for l in datasetNameList:
    count += 1

MergeFilesets(versionList,datasetNameList,skimNameList,inputPath,outputPath,filenameHeader)
Revision:	1.2
Committed:	Thu Mar 31 21:12:42 2011 UTC (14 years, 1 month ago) by bendavid
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	Mit_025c_branch2, Mit_025c_branch1, Mit_025c_branch0, Mit_025d, Mit_025c, Mit_025b, Mit_025a, Mit_025, Mit_025pre2, Mit_024b, Mit_025pre1, Mit_024a, Mit_024, Mit_023, Mit_022a, Mit_022, Mit_020d, TMit_020d, Mit_020c, Mit_021, Mit_021pre2, Mit_021pre1, Mit_020b
Branch point for:	Mit_025c_branch
Changes since 1.1:	+12 -1 lines
Log Message:	skip non-existing files gracefully
#	User	Rev	Content
1	paus	1.1	#!/usr/bin/env python
2			#===================================================================================================
3			import sys, getopt, os, fnmatch, commands
4			import string
5
6			# declare
7			setupCmsswCommand = \
8			'cd /home/$USER/cms/cmssw/017/CMSSW_3_9_5_patch1/src;eval `scram runtime -sh`;cd - >& /dev/null;'
9
10			#===================================================================================================
11			def usage():
12			print "possible options are: --help, --InputPath=<myInputPath>, --OutputPath=<myOutputPath>," \
13			+ " --FilenameHeader=<myFileHeader>, --DatasetListFile=<datasetListFile>"
14
15			#===================================================================================================
16			def filesExist(path,filenameExpression):
17			exists = False
18	bendavid	1.2
19			if not os.path.exists(path):
20			return exists;
21
22	paus	1.1	for fileName in os.listdir(path):
23			if fnmatch.fnmatch (fileName,filenameExpression):
24			exists = True
25			return exists
26
27			#===================================================================================================
28			# Merge Filesets
29			#===================================================================================================
30			def MergeFilesets(versionList,datasetNameList,skimNameList,inputPath,outputPath,filenameHeader):
31			n = 0
32			for dataset in datasetNameList:
33			#print "================================================================"
34			print " Merging files for dataset: " + dataset
35
36			# create the output directory in case it is not yet there
37			command = 'mkdir -p ' + outputPath
38			os.system(command)
39
40			outputMergedFilename = filenameHeader + '_' + dataset + '_' + skimNameList[n] + '.root'
41			inputFilenameRegExp = filenameHeader + '_' + dataset + '_' + skimNameList[n] + '_????.root'
42	bendavid	1.2	# command = setupCmsswCommand + 'hadd -f ' + outputPath + outputMergedFilename \
43			command = 'hadd -f7 ' + outputPath + outputMergedFilename \
44	paus	1.1	+ ' ' + inputPath + '/' + versionList[n] + '/' + dataset + '/' \
45			+ inputFilenameRegExp + ' >& ./merging.tmp'
46			# command = 'hadd -f ' + outputPath + \
47			# outputMergedFilename + ' ' + inputPath + inputFilenameRegExp + \
48			# ' >& ./merging.tmp'
49
50			if (filesExist(inputPath+'/'+versionList[n]+'/'+dataset,inputFilenameRegExp) == True):
51			if (os.path.exists(outputPath+outputMergedFilename)):
52			print " Warning: merged file already exists. It will be deleted.\n " + \
53			outputPath+outputMergedFilename
54			os.system('rm ' + outputPath+outputMergedFilename)
55			#print ' merging: ' + command
56			os.system(command)
57			#print ''
58			else:
59			print " Warning: No files for dataset " + dataset + "\n at the location: " + inputPath \
60			+ '/' + versionList[n] + '/' + dataset + '/' + inputFilenameRegExp
61			#print ''
62			n += 1
63
64			#===================================================================================================
65			# Main Program
66			#===================================================================================================
67			datasetListFile = ''
68			inputPath = ''
69			outputPath = ''
70			filenameHeader = ''
71			versionList = list()
72			datasetNameList = list()
73			skimNameList = list()
74
75			if len(sys.argv[1:]) < 1:
76			print "Error: not enough parameters specified"
77			usage()
78			sys.exit()
79
80			try:
81			opts, args = getopt.getopt(sys.argv[1:], "hi:o:f:d:",
82			["help","InputPath=","OutputPath=",
83			"FilenameHeader=","DatasetListFile="])
84			for o, a in opts:
85			if o in ("-h", "--help"):
86			usage()
87			sys.exit()
88			elif o in ("-i", "--InputPath"):
89			inputPath = a + "/"
90			elif o in ("-o", "--OutputPath"):
91			outputPath = a + "/"
92			elif o in ("-f", "--FilenameHeader"):
93			filenameHeader = a
94			elif o in ("-d", "--DatasetListFile"):
95			datasetListFile = a
96			else:
97			usage()
98			sys.exit()
99			except getopt.GetoptError:
100			usage()
101			sys.exit(2)
102
103			if (inputPath == ''):
104			print "Error: No InputPath specified."
105			sys.exit()
106
107			if (outputPath == ''):
108			print "Error: No OutputPath specified."
109			sys.exit()
110
111			if (filenameHeader == ''):
112			print "Error: No FilenameHeader specified."
113			sys.exit()
114
115			if (datasetListFile == ''):
116			print "Error: No dataset list file specified."
117			sys.exit()
118
119			try:
120			inputFile = open(datasetListFile,"r")
121			except IOError:
122			print "Error: The specified dataset list file " + datasetListFile + " could not be opened."
123			sys.exit()
124
125	bendavid	1.2	print datasetListFile
126	paus	1.1	#===================================================================================================
127			# Read in list of datasets and skim names
128			#===================================================================================================
129			lineNumber = 1
130			templine = inputFile.readline()
131			while len(templine.split()) > 0:
132
133	bendavid	1.2	# print "reading line"
134			# print templine
135
136	paus	1.1	# ignore commented lines
137			if (templine[0] == '#'):
138			templine = inputFile.readline()
139			lineNumber += 1
140			continue;
141
142	bendavid	1.2
143			# print "processing line"
144	paus	1.1	# check what type of list was provided and assume 'noskim' as default
145			if (len(templine.split()) == 7) :
146			tempInputList = templine.split()
147			versionList .append(tempInputList[0])
148			datasetNameList.append(tempInputList[1])
149			skimNameList .append('noskim')
150			lineNumber += 1
151			else:
152			print " ERROR: incorrect format for cross section file. Check line %s" % lineNumber
153			sys.exit()
154
155			# read the next line
156			templine = inputFile.readline()
157
158			inputFile.close()
159
160			# Check the list of variables
161			count = 0
162			for l in datasetNameList:
163			count += 1
164
165			MergeFilesets(versionList,datasetNameList,skimNameList,inputPath,outputPath,filenameHeader)