EarlyDataStudy/scripts/heds.sh

#!/bin/bash

######################################################
# heds 0 (create new directories)
# heds 1 (query runregistry)
# heds 2 (create crab task)
# heds 3 (get and publish jobs)
# heds 4 (create all the html tables and store logs)
# heds 5* (event display stuff)
######################################################

choice=$1

dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT


case $choice in
    0)
        echo "=== you selected option 0. Setting up directories and one-time things ==="
        echo ""

        ### create local folder
        if [ -d $HEDS_LOCALSTORE ] 
        then echo "directory " $HEDS_LOCALSTORE "already exists. Then skipping \"mkdir\" "
        else
            echo "making " $HEDS_LOCALSTORE " directory"
            mkdir -p $HEDS_LOCALSTORE
        fi

        ### create WWW folders
        if [ -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION ]
        then echo "directory " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION \
            " already exists. Then skipping \"mkdir\" "

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi

        else
            echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION " directory"
            mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/       

            if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
                echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA." 
                echo "Copying base html tables from there."

                ##TO BE FIXED. Check that files exist before copy
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
            fi

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi
        fi


        ### create CASTOR folder
        nsls -d $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION >& tmp
        check=`cat tmp |grep "No such" |wc -w`
        rm tmp
        if [ $check == 0 ]
        then echo "directory " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION "already exists"           
        else
            echo "making " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION " directory"
            rfmkdir -p $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION            
        fi
        ;;

    1)
        echo "=== you selected option 1. Querying RunRegistry, select good runs and submit crab jobs ==="
        echo ""

        dateForDbsSearch=`echo $HEDS_LABEL|awk -F\. '{print $3"-"$2"-"$1}'`
        echo "dateForDbsSearch: " $dateForDbsSearch
        dbs search --query="find run where dataset=$HEDS_INPUTDATASET and run.createdate=$dateForDbsSearch" \
            > tmpRunList.txt    
        lastRun=`cat tmpRunList.txt |grep [0-9][0-9][0-9][0-9][0-9][0-9] |head -n1`
        firstRun=`cat tmpRunList.txt|tail -n1`
        rm tmpRunList.txt
        #TO BE FIXED: add a protection for when there are no runs
        echo "first,last runs: " $firstRun $lastRun


        runselection="runselection="
        cat $HEDSPATH/scripts/runreg.cfg | sed s/SET_RUNMIN/$firstRun/ | \
            sed s/SET_RUNMAX/$lastRun/ > runreg.cfg
        python $HEDSPATH/scripts/runregparse.py  > tmp.log
        rm runreg.cfg
        echo "here we have the good run list"

        IFS=$'\n'
        found=0
        lastGoodRun=0
        for line in $(cat tmp.log);
        do
            if [ $found == 0 ]
            then
                found=`echo $line |grep lumisToProcess|wc |awk '{print $1}'`
                continue;
            fi
            if [ $line == ")" ] 
            then  
                break;
            fi
            thisRun=`echo $line | sed s/\'//g | sed s/,//g |awk -F: '{print $1}'`
            if [ $thisRun != $lastGoodRun ] 
            then
                runselection=$runselection$thisRun\,
            fi
            lastGoodRun=$thisRun
        done
        #echo "========= THE FOLLOWING LINE TO YOUR crab.cfg FILE ========="
        echo $runselection
        #echo "================================================================"
        echo $runselection > $HEDS_LABEL.runselection
        echo $firstRun-$lastRun > $HEDS_LABEL.runinterval
        rm tmp.log

        check=`echo $runselection | sed s/runselection=// |wc -w`

        if [ $check == 0 ] 
        then
            echo "No good runs have been selected. No CRAB cfg will be created."
        else
            echo "Some good runs have been selected. CRAB cfg will be created."
            cat $HEDSPATH/scripts/crab.template.cfg |sed "s#SET_DATASET#$HEDS_INPUTDATASET#" | \
#               sed "s#SET_REMOTEDIR#$HEDS_USERID/HEDS/#" | \
                sed "s#SET_REMOTEDIR#$HEDS_USERID/#" | \
                sed "s#SET_T2#$HEDS_T2SITE#" | \
                sed "s/SET_RUNSELECTION/$runselection/" > tmp.cfg            
            pset=$HEDSPATH/python/earlyDataInterestingEvents_cfg.py
            echo "pset: " $pset
            cat tmp.cfg |sed s#SET_PSET#$pset# | sed "s#SET_TASK_NAME#$dailySkim#" > tmp2.cfg; rm tmp.cfg
            mv tmp2.cfg $HEDS_LABEL.crab.cfg
        fi
        ;;
    2)
        echo "creating and submitting CRAB jobs for " $dailySkim
        if [ -e $HEDS_LABEL.crab.cfg ] 
        then 
            crab -create -submit -cfg $HEDS_LABEL.crab.cfg
            if ((! $?)); then rm $HEDS_LABEL.crab.cfg; fi
        else
            echo "ERROR: " $HEDS_LABEL.crab.cfg " doesn't exist"
        fi
        ;;
    3)
        echo "getting output and publishing crab jobs"
        crab -status -c $dailySkim
        crab -get -c $dailySkim 
        crab -status -c $dailySkim
        crab -publish -c $dailySkim 
        ;;
    

    4)  
        echo "option 4 "
        if [ -d $dailySkim ] #the crab job was run on at least 1 run
        then
            heds 4a 
            heds 4b 
            heds 4c 
            heds 4d 
            heds 4e 
            heds 4f 
            heds 4g 
            heds 4h 
            heds 4i 
        else
            echo "copying only the log files about runs and selected-runs"
            heds 4g
        fi
        ;;
    
    4a) 
        #### echo "...copying log files"
        if [ ! -d $HEDS_LOCALSTORE/$dailySkim/logs ]; then
            mkdir -p $HEDS_LOCALSTORE/$dailySkim/logs
        fi
        cp $dailySkim/res/*.std* $HEDS_LOCALSTORE/$dailySkim/logs
        if (($?)); then
            echo "ERROR: failed to copy log files in " $HEDS_LOCALSTORE/$dailySkim/logs;
        else
            echo "copied all logs file in " $HEDS_LOCALSTORE/$dailySkim/logs
        fi

        ###
        tarName=$HEDS_LABEL.log.tgz
        cd $HEDS_LOCALSTORE/$dailySkim/logs/
        if (($?));then
            echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
        else
            echo "compriming log files... "
            tar czvf $tarName *.std* >& /dev/null
            echo -e "done \n"

            echo "copying " $tarName " file to CASTOR area.."
            rfcp $tarName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            echo -e "done \n"
            cd -
        fi
        ;;
    
    4b) 
        ###
        echo -e "creating skimReport..."
        logName=$HEDS_LABEL.log
        $HEDSPATH/scripts/skimreport $HEDS_LOCALSTORE/$dailySkim/logs/*.stdout  > $logName
        mv $logName $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents/            
        echo -e "done \n"
        ;;
        
    4c) 
        ###
        echo "...coping files from the T2's SE"
        echo "dataset to retreive: " $dailySkim
        dbs search --url=https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet \
            --query="find file,site where dataset=*$dailySkim*" |grep .root >listFiles

        IFS=$'\n'
        for j in $(cat listFiles)
        do
            echo "going to copy file: $j"
            fileName=`echo $j |awk '{print$1}'| awk -F$dailySkim/ '{print $2}' | awk -F/ '{print $2}'`

            storage_elem=`echo $j |awk '{print $2}'`
            remote_path=`echo $j|awk '{print $1}'`
            lcg-cp -b -D srmv2  srm://$storage_elem:$HEDS_STORAGEPORT$HEDS_STORAGEPATH$remote_path file://$HEDS_LOCALSTORE/$dailySkim/$fileName
        done
        rm listFiles
        nFiles=`ls $HEDS_LOCALSTORE/$dailySkim/*.root|wc -l`
        nJobs=`ls $HEDS_LOCALSTORE/$dailySkim/logs/CMSSW_*.stdout |wc -l`

        ### TO BE FIXED!! this check isn't valid if some job selects zero events (no file is created)
        #if [ $nFiles == $nJobs ] 
        #then
        #    echo "GOOD: #files copied from hadoop matches #jobs in the CRAB task"
        #else
        #    echo "ERROR: #files in hadoop doesn't match #jobs in the CRAB task"
        #    exit
        #fi
        ;;

    4d)
        ###
        echo -e "Merging files edm files..."
        cp $HEDSPATH/scripts/prepareMerging.sh $HEDS_LOCALSTORE/$dailySkim
        cp $HEDSPATH/scripts/merge.py $HEDS_LOCALSTORE/$dailySkim
        cd $HEDS_LOCALSTORE/$dailySkim

        ./prepareMerging.sh 
        numberFilesToMerge=`ls *.root|wc -l`
        echo "numberFilesToMerge: " $numberFilesToMerge
        cmsRun merge.py >& merging.log 
        echo -e "done \n"
        cd -
        ;;
    
    4e) 
        ###
        echo -e "Storing merged edm file in CASTOR..."
        mergedName=$HEDS_LABEL.root
        mergedName=merged_$mergedName
        echo "mergedName: " $mergedName
        cd $HEDS_LOCALSTORE/$dailySkim
        if ((! $?)); then
            mv merged.root ../$mergedName
            echo "going to copy the merged file into: " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            rfcp ../$mergedName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION/          
            cd -
        fi
        echo -e "done \n"
        ;;
    
    4f) 
        ###
        echo "Preparing detailed list of skimmed events..."
        cp $HEDSPATH/scripts/listSkimmedEvent.C $HEDS_LOCALSTORE            
        cd $HEDS_LOCALSTORE
        root.exe -b -q "listSkimmedEvent.C+(\"$HEDS_LABEL\")" >& /dev/null
        listSkimmedEventsPath=$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        if [ ! -d $listSkimmedEventsPath ]
        then
            mkdir $listSkimmedEventsPath
        fi
        listDetailedLogs=`ls $HEDS_LABEL*`
        for j in $listDetailedLogs
        do
            echo "preparing log " $j
            # sort entries by date
            cat $j |sort -k7 > tmp
            mv tmp $j
            mv $j $listSkimmedEventsPath
        done
        echo -e "done \n"
        cd -
        ;;
    
    4g)
        ###
        echo "Copying logs about run selection..."
        mv $HEDS_LABEL.runselection $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        mv $HEDS_LABEL.runinterval $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        echo -e "done \n"
        ;;

    4h)
        ###
        echo "Preparing event dumps..."
        input=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "input: " $input
        sed s#INPUT#$input# $HEDSPATH/scripts/dumpEvent.py > dumpEvent.py
        cmsRun dumpEvent.py >& $HEDS_LABEL.dumperLog
        $HEDSPATH/scripts/parseDumperOutput.sh 1 >& $HEDS_LABEL.dump
        $HEDSPATH/scripts/parseDumperOutput.sh 2 
        rm dumpEvent.py
        rm $HEDS_LABEL.dumperLog
        rm $HEDS_LABEL.dump
        echo -e "done \n"
        ;;
    4i)
        ###
        echo "Preparing html tables..."
        if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
            echo "create first version of FirstTable.sh"
            $HEDSPATH/scripts/editFirstTable.sh create
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
        else
            $HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
        fi
        echo -e "done \n"
        ;;

    4j) ###
        echo -e "Your preliminary table is available at the following URL: "
        url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/pagesInPreliminaryShape"
        url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
        echo -e $url "\n"
        echo -e "After you verified that all the tables have been correctly produced, " \
            "they can be moved to the final repository \n"

        ;;

    5a)
        echo ".... preparing event display server"
        mkdir $HEDS_LABEL.shots
        port=3005
        $HEDSPATH/scripts/eventDisplayServer.sh $port &
        ;;
    5b)
        echo "getting snapshots"
        port=3005
        file=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "$file" | nc -w 10 localhost $port
        ;;
    5c)
        echo "moving snapshots"
        mv $HEDS_LABEL.shots $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
        cp $HEDSPATH/scripts/indexFileForDetailedSnapShots.php \
            $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.shots/index.php
        ;;


    final)
        #TO BE FIXED. this call could written in a  much more compact way
        echo "WWW pages are going to be moved from the temporary repository"
        echo -e "("$HEDS_WWWAREA_TMP")\n"
        echo "to the permanent one"
        echo -e "("$HEDS_WWWAREA")\n"

        if [ ! -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
            mkdir $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION
        fi
        
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm.BAK
        fi
        
        mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
        mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/

        mv -T $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents


        url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/"
        url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
        
        echo "The new tables created by you are accessible through the following URL: "
        echo $url
        ;;

    #####################  after the crab part is done ###################
    99) echo "Doing bakcup of $HEDS_WWWAREA_TMP"
        date=`date +%k\.%M\.%S_%e\.%m\.%Y`
        fileName="HiggsIES.BAK."$date".tgz"
        echo "file name: " $fileName
        tar -czvf $fileName $HEDS_WWWAREA_TMP
        mv $fileName $HEDS_WWWAREA_TMP/..
        ;;


    *)
        echo "option not defined"
        ;;
esac
Revision:	1.4
Committed:	Fri May 7 14:27:01 2010 UTC (14 years, 11 months ago) by mangano
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	V00-00-01
Changes since 1.3:	+2 -0 lines
Log Message:	minor changes
#	User	Rev	Content
1	mangano	1.1	#!/bin/bash
2
3			######################################################
4			# heds 0 (create new directories)
5			# heds 1 (query runregistry)
6			# heds 2 (create crab task)
7			# heds 3 (get and publish jobs)
8			# heds 4 (create all the html tables and store logs)
9			# heds 5* (event display stuff)
10			######################################################
11
12			choice=$1
13
14			dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT
15
16
17
18			case $choice in
19			0)
20			echo "=== you selected option 0. Setting up directories and one-time things ==="
21			echo ""
22
23	mangano	1.3	### create local folder
24	mangano	1.1	if [ -d $HEDS_LOCALSTORE ]
25	mangano	1.3	then echo "directory " $HEDS_LOCALSTORE "already exists. Then skipping \"mkdir\" "
26	mangano	1.1	else
27			echo "making " $HEDS_LOCALSTORE " directory"
28			mkdir -p $HEDS_LOCALSTORE
29			fi
30
31	mangano	1.3	### create WWW folders
32	mangano	1.2	if [ -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION ]
33	mangano	1.3	then echo "directory " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION \
34			" already exists. Then skipping \"mkdir\" "
35
36			if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
37			echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
38			mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
39			fi
40
41	mangano	1.1	else
42	mangano	1.2	echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION " directory"
43	mangano	1.3	mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
44
45			if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
46			echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA."
47			echo "Copying base html tables from there."
48	mangano	1.4
49			##TO BE FIXED. Check that files exist before copy
50	mangano	1.3	cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
51			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
52			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
53			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
54			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
55			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
56			fi
57
58			if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
59			echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
60			mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
61			fi
62	mangano	1.1	fi
63	mangano	1.3
64
65			### create CASTOR folder
66	mangano	1.1	nsls -d $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION >& tmp
67			check=`cat tmp \|grep "No such" \|wc -w`
68			rm tmp
69			if [ $check == 0 ]
70			then echo "directory " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION "already exists"
71			else
72			echo "making " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION " directory"
73			rfmkdir -p $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
74			fi
75			;;
76
77			1)
78			echo "=== you selected option 1. Querying RunRegistry, select good runs and submit crab jobs ==="
79			echo ""
80
81			dateForDbsSearch=`echo $HEDS_LABEL\|awk -F\. '{print $3"-"$2"-"$1}'`
82			echo "dateForDbsSearch: " $dateForDbsSearch
83			dbs search --query="find run where dataset=$HEDS_INPUTDATASET and run.createdate=$dateForDbsSearch" \
84			> tmpRunList.txt
85			lastRun=`cat tmpRunList.txt \|grep [0-9][0-9][0-9][0-9][0-9][0-9] \|head -n1`
86			firstRun=`cat tmpRunList.txt\|tail -n1`
87			rm tmpRunList.txt
88			#TO BE FIXED: add a protection for when there are no runs
89			echo "first,last runs: " $firstRun $lastRun
90
91
92			runselection="runselection="
93			cat $HEDSPATH/scripts/runreg.cfg \| sed s/SET_RUNMIN/$firstRun/ \| \
94			sed s/SET_RUNMAX/$lastRun/ > runreg.cfg
95			python $HEDSPATH/scripts/runregparse.py > tmp.log
96			rm runreg.cfg
97			echo "here we have the good run list"
98
99			IFS=$'\n'
100			found=0
101			lastGoodRun=0
102			for line in $(cat tmp.log);
103			do
104			if [ $found == 0 ]
105			then
106			found=`echo $line \|grep lumisToProcess\|wc \|awk '{print $1}'`
107			continue;
108			fi
109			if [ $line == ")" ]
110			then
111			break;
112			fi
113			thisRun=`echo $line \| sed s/\'//g \| sed s/,//g \|awk -F: '{print $1}'`
114			if [ $thisRun != $lastGoodRun ]
115			then
116			runselection=$runselection$thisRun\,
117			fi
118			lastGoodRun=$thisRun
119			done
120			#echo "========= THE FOLLOWING LINE TO YOUR crab.cfg FILE ========="
121			echo $runselection
122			#echo "================================================================"
123			echo $runselection > $HEDS_LABEL.runselection
124			echo $firstRun-$lastRun > $HEDS_LABEL.runinterval
125			rm tmp.log
126
127			check=`echo $runselection \| sed s/runselection=// \|wc -w`
128
129			if [ $check == 0 ]
130			then
131			echo "No good runs have been selected. No CRAB cfg will be created."
132			else
133			echo "Some good runs have been selected. CRAB cfg will be created."
134			cat $HEDSPATH/scripts/crab.template.cfg \|sed "s#SET_DATASET#$HEDS_INPUTDATASET#" \| \
135	mangano	1.3	# sed "s#SET_REMOTEDIR#$HEDS_USERID/HEDS/#" \| \
136			sed "s#SET_REMOTEDIR#$HEDS_USERID/#" \| \
137	mangano	1.1	sed "s#SET_T2#$HEDS_T2SITE#" \| \
138			sed "s/SET_RUNSELECTION/$runselection/" > tmp.cfg
139			pset=$HEDSPATH/python/earlyDataInterestingEvents_cfg.py
140			echo "pset: " $pset
141			cat tmp.cfg \|sed s#SET_PSET#$pset# \| sed "s#SET_TASK_NAME#$dailySkim#" > tmp2.cfg; rm tmp.cfg
142			mv tmp2.cfg $HEDS_LABEL.crab.cfg
143			fi
144			;;
145			2)
146			echo "creating and submitting CRAB jobs for " $dailySkim
147			if [ -e $HEDS_LABEL.crab.cfg ]
148			then
149			crab -create -submit -cfg $HEDS_LABEL.crab.cfg
150			if ((! $?)); then rm $HEDS_LABEL.crab.cfg; fi
151			else
152			echo "ERROR: " $HEDS_LABEL.crab.cfg " doesn't exist"
153			fi
154			;;
155			3)
156			echo "getting output and publishing crab jobs"
157			crab -status -c $dailySkim
158			crab -get -c $dailySkim
159			crab -status -c $dailySkim
160			crab -publish -c $dailySkim
161			;;
162
163
164			4)
165			echo "option 4 "
166			if [ -d $dailySkim ] #the crab job was run on at least 1 run
167			then
168			heds 4a
169			heds 4b
170			heds 4c
171			heds 4d
172			heds 4e
173			heds 4f
174			heds 4g
175			heds 4h
176			heds 4i
177			else
178			echo "copying only the log files about runs and selected-runs"
179			heds 4g
180			fi
181			;;
182
183			4a)
184	mangano	1.3	#### echo "...copying log files"
185	mangano	1.1	if [ ! -d $HEDS_LOCALSTORE/$dailySkim/logs ]; then
186			mkdir -p $HEDS_LOCALSTORE/$dailySkim/logs
187			fi
188			cp $dailySkim/res/.std $HEDS_LOCALSTORE/$dailySkim/logs
189			if (($?)); then
190			echo "ERROR: failed to copy log files in " $HEDS_LOCALSTORE/$dailySkim/logs;
191			else
192			echo "copied all logs file in " $HEDS_LOCALSTORE/$dailySkim/logs
193			fi
194
195			###
196			tarName=$HEDS_LABEL.log.tgz
197			cd $HEDS_LOCALSTORE/$dailySkim/logs/
198			if (($?));then
199			echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
200			else
201	mangano	1.3	echo "compriming log files... "
202			tar czvf $tarName .std >& /dev/null
203			echo -e "done \n"
204
205			echo "copying " $tarName " file to CASTOR area.."
206	mangano	1.1	rfcp $tarName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
207	mangano	1.3	echo -e "done \n"
208	mangano	1.1	cd -
209			fi
210			;;
211
212			4b)
213			###
214	mangano	1.3	echo -e "creating skimReport..."
215	mangano	1.1	logName=$HEDS_LABEL.log
216			$HEDSPATH/scripts/skimreport $HEDS_LOCALSTORE/$dailySkim/logs/*.stdout > $logName
217	mangano	1.3	mv $logName $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents/
218			echo -e "done \n"
219	mangano	1.1	;;
220
221			4c)
222			###
223			echo "...coping files from the T2's SE"
224			echo "dataset to retreive: " $dailySkim
225			dbs search --url=https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet \
226			--query="find file,site where dataset=$dailySkim" \|grep .root >listFiles
227
228			IFS=$'\n'
229			for j in $(cat listFiles)
230			do
231			echo "going to copy file: $j"
232			fileName=`echo $j \|awk '{print$1}'\| awk -F$dailySkim/ '{print $2}' \| awk -F/ '{print $2}'`
233
234			storage_elem=`echo $j \|awk '{print $2}'`
235			remote_path=`echo $j\|awk '{print $1}'`
236			lcg-cp -b -D srmv2 srm://$storage_elem:$HEDS_STORAGEPORT$HEDS_STORAGEPATH$remote_path file://$HEDS_LOCALSTORE/$dailySkim/$fileName
237			done
238			rm listFiles
239			nFiles=`ls $HEDS_LOCALSTORE/$dailySkim/*.root\|wc -l`
240			nJobs=`ls $HEDS_LOCALSTORE/$dailySkim/logs/CMSSW_*.stdout \|wc -l`
241
242			### TO BE FIXED!! this check isn't valid if some job selects zero events (no file is created)
243			#if [ $nFiles == $nJobs ]
244			#then
245			# echo "GOOD: #files copied from hadoop matches #jobs in the CRAB task"
246			#else
247			# echo "ERROR: #files in hadoop doesn't match #jobs in the CRAB task"
248			# exit
249			#fi
250			;;
251
252			4d)
253			###
254	mangano	1.3	echo -e "Merging files edm files..."
255	mangano	1.1	cp $HEDSPATH/scripts/prepareMerging.sh $HEDS_LOCALSTORE/$dailySkim
256			cp $HEDSPATH/scripts/merge.py $HEDS_LOCALSTORE/$dailySkim
257			cd $HEDS_LOCALSTORE/$dailySkim
258
259			./prepareMerging.sh
260			numberFilesToMerge=`ls *.root\|wc -l`
261			echo "numberFilesToMerge: " $numberFilesToMerge
262			cmsRun merge.py >& merging.log
263	mangano	1.3	echo -e "done \n"
264	mangano	1.1	cd -
265			;;
266
267			4e)
268			###
269	mangano	1.3	echo -e "Storing merged edm file in CASTOR..."
270	mangano	1.1	mergedName=$HEDS_LABEL.root
271			mergedName=merged_$mergedName
272			echo "mergedName: " $mergedName
273			cd $HEDS_LOCALSTORE/$dailySkim
274			if ((! $?)); then
275			mv merged.root ../$mergedName
276			echo "going to copy the merged file into: " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
277			rfcp ../$mergedName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION/
278			cd -
279			fi
280	mangano	1.3	echo -e "done \n"
281	mangano	1.1	;;
282
283			4f)
284			###
285	mangano	1.3	echo "Preparing detailed list of skimmed events..."
286	mangano	1.1	cp $HEDSPATH/scripts/listSkimmedEvent.C $HEDS_LOCALSTORE
287			cd $HEDS_LOCALSTORE
288	mangano	1.3	root.exe -b -q "listSkimmedEvent.C+(\"$HEDS_LABEL\")" >& /dev/null
289	mangano	1.2	listSkimmedEventsPath=$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
290	mangano	1.1	if [ ! -d $listSkimmedEventsPath ]
291			then
292			mkdir $listSkimmedEventsPath
293			fi
294			listDetailedLogs=`ls $HEDS_LABEL*`
295			for j in $listDetailedLogs
296			do
297			echo "preparing log " $j
298			# sort entries by date
299			cat $j \|sort -k7 > tmp
300			mv tmp $j
301			mv $j $listSkimmedEventsPath
302			done
303	mangano	1.3	echo -e "done \n"
304	mangano	1.1	cd -
305			;;
306
307			4g)
308			###
309	mangano	1.3	echo "Copying logs about run selection..."
310			mv $HEDS_LABEL.runselection $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
311			mv $HEDS_LABEL.runinterval $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
312			echo -e "done \n"
313	mangano	1.1	;;
314
315			4h)
316			###
317	mangano	1.3	echo "Preparing event dumps..."
318	mangano	1.1	input=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
319			echo "input: " $input
320			sed s#INPUT#$input# $HEDSPATH/scripts/dumpEvent.py > dumpEvent.py
321			cmsRun dumpEvent.py >& $HEDS_LABEL.dumperLog
322			$HEDSPATH/scripts/parseDumperOutput.sh 1 >& $HEDS_LABEL.dump
323			$HEDSPATH/scripts/parseDumperOutput.sh 2
324			rm dumpEvent.py
325			rm $HEDS_LABEL.dumperLog
326			rm $HEDS_LABEL.dump
327	mangano	1.3	echo -e "done \n"
328	mangano	1.1	;;
329			4i)
330			###
331	mangano	1.3	echo "Preparing html tables..."
332	mangano	1.2	if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
333	mangano	1.1	echo "create first version of FirstTable.sh"
334			$HEDSPATH/scripts/editFirstTable.sh create
335	mangano	1.3	$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
336			$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
337			$HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
338			else
339			$HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
340			$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
341			$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
342			$HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
343			fi
344			echo -e "done \n"
345			;;
346
347			4j) ###
348			echo -e "Your preliminary table is available at the following URL: "
349			url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/pagesInPreliminaryShape"
350			url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
351			echo -e $url "\n"
352			echo -e "After you verified that all the tables have been correctly produced, " \
353			"they can be moved to the final repository \n"
354
355	mangano	1.1	;;
356	mangano	1.3
357	mangano	1.1	5a)
358			echo ".... preparing event display server"
359			mkdir $HEDS_LABEL.shots
360			port=3005
361			$HEDSPATH/scripts/eventDisplayServer.sh $port &
362			;;
363			5b)
364			echo "getting snapshots"
365			port=3005
366			file=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
367			echo "$file" \| nc -w 10 localhost $port
368			;;
369			5c)
370			echo "moving snapshots"
371	mangano	1.2	mv $HEDS_LABEL.shots $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
372	mangano	1.1	cp $HEDSPATH/scripts/indexFileForDetailedSnapShots.php \
373	mangano	1.2	$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.shots/index.php
374	mangano	1.1	;;
375
376
377	mangano	1.3	final)
378			#TO BE FIXED. this call could written in a much more compact way
379			echo "WWW pages are going to be moved from the temporary repository"
380			echo -e "("$HEDS_WWWAREA_TMP")\n"
381			echo "to the permanent one"
382			echo -e "("$HEDS_WWWAREA")\n"
383
384			if [ ! -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
385			mkdir $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION
386			fi
387
388			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ]; then
389			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
390			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php.BAK
391			fi
392			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm ]; then
393			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
394			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm.BAK
395			fi
396			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.php ]; then
397			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
398			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm.BAK
399			fi
400
401			mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.php \
402			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
403			mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.shtm \
404			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
405
406			mv -T $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents \
407			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
408
409
410			url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/"
411			url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
412
413			echo "The new tables created by you are accessible through the following URL: "
414			echo $url
415			;;
416
417	mangano	1.1	##################### after the crab part is done ###################
418	mangano	1.2	99) echo "Doing bakcup of $HEDS_WWWAREA_TMP"
419	mangano	1.1	date=`date +%k\.%M\.%S_%e\.%m\.%Y`
420			fileName="HiggsIES.BAK."$date".tgz"
421			echo "file name: " $fileName
422	mangano	1.2	tar -czvf $fileName $HEDS_WWWAREA_TMP
423			mv $fileName $HEDS_WWWAREA_TMP/..
424	mangano	1.1	;;
425
426
427			*)
428			echo "option not defined"
429			;;
430			esac