PRODREQUEST/Applications/ProdRequestUtils.py

import os
import sys
import getopt
import popen2
import time


# Fixme: do the same for createProdWorkflow
def createPreProdWorkflow (PyCfg = None,
                           PSetHash = None,
                           prodName = None,
                           cmsswVersion = None,
                           category = "PreProd",
                           timestamp = int(time.time()),
                           FakeHash = False,
                           pileupDS = None,
                           pileupFilesPerJob = 1,
                           dbsAddress = None,
                           dbsUrl = None,
                           dlsAddress = None,
                           dlsType = None,
                           pileupSkipLocation = False):
    spec = WorkflowSpec()
    spec.setWorkflowName(prodName)
    spec.setRequestCategory(category)
    spec.setRequestTimestamp(timestamp)
    
    # //
    # // This value was created by running the EdmConfigHash tool
    # //  on the original cfg file.
    
    
    cmsRun = spec.payload
    cmsRun.name = "cmsRun1" # every node in the workflow needs a unique name
    cmsRun.type = "CMSSW"   # Nodes that are CMSSW based should set the name
    cmsRun.application["Project"] = "CMSSW" # project
    cmsRun.application["Version"] = cmsswVersion # version
    cmsRun.application["Architecture"] = "slc3_ia32_gcc323" # arch (not needed)
    cmsRun.application["Executable"] = "cmsRun" # binary name
    cmsRun.configuration = PyCfg # Python PSet file
    
    
    # //
    # // Pileup sample?
    # //
    if pileupDS != None:
        puPrimary = pileupDS.split("/")[1]
        puTier = pileupDS.split("/")[2]
        puProc = pileupDS.split("/")[3]
        puDataset = cmsRun.addPileupDataset(puPrimary, puTier, puProc)
        puDataset['FilesPerJob'] = pileupFilesPerJob
        if customDBSDLS:
            puDataset['DBSAddress'] = dbsAddress
            puDataset['DBSURL'] = dbsUrl
            puDataset['DLSType'] = dlsType
            puDataset['DLSAddress'] = dlsAddress
        if pileupSkipLocation:
            puDataset['SkipLocation'] = pileupSkipLocation 
    
    #  //
    # // Pull all the output modules from the configuration file,
    #//  treat the output module name as DataTier and AppFamily,
    #  //since there isnt a standard way to derive these things yet.
    # //    
    #//  For each module a dataset declaration is created in the spec
    cfgInt = CfgInterface(cmsRun.configuration, True)
    datasetList = []
    for outModName, val in cfgInt.outputModules.items():
        #  //
        # // Check for Multi Tiers.
        #//  If Output module contains - characters, we split based on it
        #  //And create a different tier for each basic tier
        # //
        #//
    
        tierList = splitMultiTier(outModName)    
        for dataTier in tierList:
            processedDS = "%s-%s-%s-unmerged" % (
                cmsRun.application['Version'], outModName, timestamp)
            outDS = cmsRun.addOutputDataset(prodName, 
                                            processedDS,
                                            outModName)
    
            outDS['DataTier'] = dataTier
            outDS["ApplicationName"] = cmsRun.application["Executable"]
            outDS["ApplicationProject"] = cmsRun.application["Project"]
            outDS["ApplicationVersion"] = cmsRun.application["Version"]
            outDS["ApplicationFamily"] = outModName
            if FakeHash:
                guid = MCPayloadsUUID.uuidgen()
                if guid == None:
                    guid = MCPayloadsUUID.uuid()
                hashValue = "hash=%s;guid=%s" % (PSetHash, guid)
                outDS['PSetHash'] = hashValue
            else:
                outDS['PSetHash'] = PSetHash
            datasetList.append(outDS.name())
    
    stageOut = cmsRun.newNode("stageOut1")
    stageOut.type = "StageOut"
    stageOut.application["Project"] = ""
    stageOut.application["Version"] = ""
    stageOut.application["Architecture"] = ""
    stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
    stageOut.configuration = ""
    
    mergedLFNBase(spec)
    unmergedLFNBase(spec)
    
    return spec
Revision:	1.4
Committed:	Fri May 25 07:07:27 2007 UTC (17 years, 11 months ago) by eulisse
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	V00-08-00, V00-07-09, V00-07-08, V00-07-07, V00-07-06, V00-07-05, V00-07-04, V00-07-03, V00-07-02, V00-07-01, V00-07-00, gePreProdManagerSupport, V00-06-07, V00-06-06, V00-06-05, V00-06-04, HEAD
Changes since 1.3:	+0 -18 lines
Log Message:	* New way of creating workflows. Now prodcommon is actually decoupled from prodrequest and the workflow creation happens in a separate script. Different scripts are now registered using the plugin mechanism. * WorkflowEntry now has a column for cmssw_version and prodcommon_version.
#	User	Rev	Content
1	eulisse	1.1	import os
2			import sys
3			import getopt
4			import popen2
5			import time
6
7
8			# Fixme: do the same for createProdWorkflow
9	eulisse	1.2	def createPreProdWorkflow (PyCfg = None,
10			PSetHash = None,
11	eulisse	1.1	prodName = None,
12	eulisse	1.2	cmsswVersion = None,
13	eulisse	1.1	category = "PreProd",
14			timestamp = int(time.time()),
15	eulisse	1.3	FakeHash = False,
16	eulisse	1.1	pileupDS = None,
17			pileupFilesPerJob = 1,
18			dbsAddress = None,
19			dbsUrl = None,
20			dlsAddress = None,
21			dlsType = None,
22			pileupSkipLocation = False):
23			spec = WorkflowSpec()
24			spec.setWorkflowName(prodName)
25			spec.setRequestCategory(category)
26			spec.setRequestTimestamp(timestamp)
27
28			# //
29			# // This value was created by running the EdmConfigHash tool
30			# // on the original cfg file.
31
32
33			cmsRun = spec.payload
34			cmsRun.name = "cmsRun1" # every node in the workflow needs a unique name
35			cmsRun.type = "CMSSW" # Nodes that are CMSSW based should set the name
36			cmsRun.application["Project"] = "CMSSW" # project
37	eulisse	1.2	cmsRun.application["Version"] = cmsswVersion # version
38	eulisse	1.1	cmsRun.application["Architecture"] = "slc3_ia32_gcc323" # arch (not needed)
39			cmsRun.application["Executable"] = "cmsRun" # binary name
40	eulisse	1.2	cmsRun.configuration = PyCfg # Python PSet file
41	eulisse	1.1
42
43			# //
44			# // Pileup sample?
45			# //
46			if pileupDS != None:
47			puPrimary = pileupDS.split("/")[1]
48			puTier = pileupDS.split("/")[2]
49			puProc = pileupDS.split("/")[3]
50			puDataset = cmsRun.addPileupDataset(puPrimary, puTier, puProc)
51			puDataset['FilesPerJob'] = pileupFilesPerJob
52			if customDBSDLS:
53			puDataset['DBSAddress'] = dbsAddress
54			puDataset['DBSURL'] = dbsUrl
55			puDataset['DLSType'] = dlsType
56			puDataset['DLSAddress'] = dlsAddress
57			if pileupSkipLocation:
58			puDataset['SkipLocation'] = pileupSkipLocation
59
60			# //
61			# // Pull all the output modules from the configuration file,
62			#// treat the output module name as DataTier and AppFamily,
63			# //since there isnt a standard way to derive these things yet.
64			# //
65			#// For each module a dataset declaration is created in the spec
66			cfgInt = CfgInterface(cmsRun.configuration, True)
67			datasetList = []
68			for outModName, val in cfgInt.outputModules.items():
69			# //
70			# // Check for Multi Tiers.
71			#// If Output module contains - characters, we split based on it
72			# //And create a different tier for each basic tier
73			# //
74			#//
75
76			tierList = splitMultiTier(outModName)
77			for dataTier in tierList:
78			processedDS = "%s-%s-%s-unmerged" % (
79			cmsRun.application['Version'], outModName, timestamp)
80			outDS = cmsRun.addOutputDataset(prodName,
81			processedDS,
82			outModName)
83
84			outDS['DataTier'] = dataTier
85			outDS["ApplicationName"] = cmsRun.application["Executable"]
86			outDS["ApplicationProject"] = cmsRun.application["Project"]
87			outDS["ApplicationVersion"] = cmsRun.application["Version"]
88			outDS["ApplicationFamily"] = outModName
89	eulisse	1.3	if FakeHash:
90	eulisse	1.1	guid = MCPayloadsUUID.uuidgen()
91			if guid == None:
92			guid = MCPayloadsUUID.uuid()
93	eulisse	1.2	hashValue = "hash=%s;guid=%s" % (PSetHash, guid)
94	eulisse	1.1	outDS['PSetHash'] = hashValue
95			else:
96	eulisse	1.2	outDS['PSetHash'] = PSetHash
97	eulisse	1.1	datasetList.append(outDS.name())
98
99			stageOut = cmsRun.newNode("stageOut1")
100			stageOut.type = "StageOut"
101			stageOut.application["Project"] = ""
102			stageOut.application["Version"] = ""
103			stageOut.application["Architecture"] = ""
104			stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
105			stageOut.configuration = ""
106
107			mergedLFNBase(spec)
108			unmergedLFNBase(spec)
109
110	eulisse	1.2	return spec