EarlyDataStudy/scripts/heds.sh

#!/bin/bash

######################################################
# heds 0 (create new directories)
# heds 1 (query runregistry)
# heds 2 (create crab task)
# heds 3 (get and publish jobs)
# heds 4 (create all the html tables and store logs)
# heds 5* (event display stuff)
######################################################

choice=$1

dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT


case $choice in
    0)
        echo "=== you selected option 0. Setting up directories and one-time things ==="
        echo ""

        ### create local folder
        if [ -d $HEDS_LOCALSTORE ] 
        then echo "directory " $HEDS_LOCALSTORE "already exists. Then skipping \"mkdir\" "
        else
            echo "making " $HEDS_LOCALSTORE " directory"
            mkdir -p $HEDS_LOCALSTORE
        fi

        ### create WWW folders
        if [ -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION ]
        then echo "directory " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION \
            " already exists. Then skipping \"mkdir\" "

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi

        else
            echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION " directory"
            mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/       

            if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
                echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA." 
                echo "Copying base html tables from there."
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
            fi

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi
        fi


        ### create CASTOR folder
        nsls -d $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION >& tmp
        check=`cat tmp |grep "No such" |wc -w`
        rm tmp
        if [ $check == 0 ]
        then echo "directory " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION "already exists"           
        else
            echo "making " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION " directory"
            rfmkdir -p $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION            
        fi
        ;;

    1)
        echo "=== you selected option 1. Querying RunRegistry, select good runs and submit crab jobs ==="
        echo ""

        dateForDbsSearch=`echo $HEDS_LABEL|awk -F\. '{print $3"-"$2"-"$1}'`
        echo "dateForDbsSearch: " $dateForDbsSearch
        dbs search --query="find run where dataset=$HEDS_INPUTDATASET and run.createdate=$dateForDbsSearch" \
            > tmpRunList.txt    
        lastRun=`cat tmpRunList.txt |grep [0-9][0-9][0-9][0-9][0-9][0-9] |head -n1`
        firstRun=`cat tmpRunList.txt|tail -n1`
        rm tmpRunList.txt
        #TO BE FIXED: add a protection for when there are no runs
        echo "first,last runs: " $firstRun $lastRun


        runselection="runselection="
        cat $HEDSPATH/scripts/runreg.cfg | sed s/SET_RUNMIN/$firstRun/ | \
            sed s/SET_RUNMAX/$lastRun/ > runreg.cfg
        python $HEDSPATH/scripts/runregparse.py  > tmp.log
        rm runreg.cfg
        echo "here we have the good run list"

        IFS=$'\n'
        found=0
        lastGoodRun=0
        for line in $(cat tmp.log);
        do
            if [ $found == 0 ]
            then
                found=`echo $line |grep lumisToProcess|wc |awk '{print $1}'`
                continue;
            fi
            if [ $line == ")" ] 
            then  
                break;
            fi
            thisRun=`echo $line | sed s/\'//g | sed s/,//g |awk -F: '{print $1}'`
            if [ $thisRun != $lastGoodRun ] 
            then
                runselection=$runselection$thisRun\,
            fi
            lastGoodRun=$thisRun
        done
        #echo "========= THE FOLLOWING LINE TO YOUR crab.cfg FILE ========="
        echo $runselection
        #echo "================================================================"
        echo $runselection > $HEDS_LABEL.runselection
        echo $firstRun-$lastRun > $HEDS_LABEL.runinterval
        rm tmp.log

        check=`echo $runselection | sed s/runselection=// |wc -w`

        if [ $check == 0 ] 
        then
            echo "No good runs have been selected. No CRAB cfg will be created."
        else
            echo "Some good runs have been selected. CRAB cfg will be created."
            cat $HEDSPATH/scripts/crab.template.cfg |sed "s#SET_DATASET#$HEDS_INPUTDATASET#" | \
#               sed "s#SET_REMOTEDIR#$HEDS_USERID/HEDS/#" | \
                sed "s#SET_REMOTEDIR#$HEDS_USERID/#" | \
                sed "s#SET_T2#$HEDS_T2SITE#" | \
                sed "s/SET_RUNSELECTION/$runselection/" > tmp.cfg            
            pset=$HEDSPATH/python/earlyDataInterestingEvents_cfg.py
            echo "pset: " $pset
            cat tmp.cfg |sed s#SET_PSET#$pset# | sed "s#SET_TASK_NAME#$dailySkim#" > tmp2.cfg; rm tmp.cfg
            mv tmp2.cfg $HEDS_LABEL.crab.cfg
        fi
        ;;
    2)
        echo "creating and submitting CRAB jobs for " $dailySkim
        if [ -e $HEDS_LABEL.crab.cfg ] 
        then 
            crab -create -submit -cfg $HEDS_LABEL.crab.cfg
            if ((! $?)); then rm $HEDS_LABEL.crab.cfg; fi
        else
            echo "ERROR: " $HEDS_LABEL.crab.cfg " doesn't exist"
        fi
        ;;
    3)
        echo "getting output and publishing crab jobs"
        crab -status -c $dailySkim
        crab -get -c $dailySkim 
        crab -status -c $dailySkim
        crab -publish -c $dailySkim 
        ;;
    

    4)  
        echo "option 4 "
        if [ -d $dailySkim ] #the crab job was run on at least 1 run
        then
            heds 4a 
            heds 4b 
            heds 4c 
            heds 4d 
            heds 4e 
            heds 4f 
            heds 4g 
            heds 4h 
            heds 4i 
        else
            echo "copying only the log files about runs and selected-runs"
            heds 4g
        fi
        ;;
    
    4a) 
        #### echo "...copying log files"
        if [ ! -d $HEDS_LOCALSTORE/$dailySkim/logs ]; then
            mkdir -p $HEDS_LOCALSTORE/$dailySkim/logs
        fi
        cp $dailySkim/res/*.std* $HEDS_LOCALSTORE/$dailySkim/logs
        if (($?)); then
            echo "ERROR: failed to copy log files in " $HEDS_LOCALSTORE/$dailySkim/logs;
        else
            echo "copied all logs file in " $HEDS_LOCALSTORE/$dailySkim/logs
        fi

        ###
        tarName=$HEDS_LABEL.log.tgz
        cd $HEDS_LOCALSTORE/$dailySkim/logs/
        if (($?));then
            echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
        else
            echo "compriming log files... "
            tar czvf $tarName *.std* >& /dev/null
            echo -e "done \n"

            echo "copying " $tarName " file to CASTOR area.."
            rfcp $tarName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            echo -e "done \n"
            cd -
        fi
        ;;
    
    4b) 
        ###
        echo -e "creating skimReport..."
        logName=$HEDS_LABEL.log
        $HEDSPATH/scripts/skimreport $HEDS_LOCALSTORE/$dailySkim/logs/*.stdout  > $logName
        mv $logName $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents/            
        echo -e "done \n"
        ;;
        
    4c) 
        ###
        echo "...coping files from the T2's SE"
        echo "dataset to retreive: " $dailySkim
        dbs search --url=https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet \
            --query="find file,site where dataset=*$dailySkim*" |grep .root >listFiles

        IFS=$'\n'
        for j in $(cat listFiles)
        do
            echo "going to copy file: $j"
            fileName=`echo $j |awk '{print$1}'| awk -F$dailySkim/ '{print $2}' | awk -F/ '{print $2}'`

            storage_elem=`echo $j |awk '{print $2}'`
            remote_path=`echo $j|awk '{print $1}'`
            lcg-cp -b -D srmv2  srm://$storage_elem:$HEDS_STORAGEPORT$HEDS_STORAGEPATH$remote_path file://$HEDS_LOCALSTORE/$dailySkim/$fileName
        done
        rm listFiles
        nFiles=`ls $HEDS_LOCALSTORE/$dailySkim/*.root|wc -l`
        nJobs=`ls $HEDS_LOCALSTORE/$dailySkim/logs/CMSSW_*.stdout |wc -l`

        ### TO BE FIXED!! this check isn't valid if some job selects zero events (no file is created)
        #if [ $nFiles == $nJobs ] 
        #then
        #    echo "GOOD: #files copied from hadoop matches #jobs in the CRAB task"
        #else
        #    echo "ERROR: #files in hadoop doesn't match #jobs in the CRAB task"
        #    exit
        #fi
        ;;

    4d)
        ###
        echo -e "Merging files edm files..."
        cp $HEDSPATH/scripts/prepareMerging.sh $HEDS_LOCALSTORE/$dailySkim
        cp $HEDSPATH/scripts/merge.py $HEDS_LOCALSTORE/$dailySkim
        cd $HEDS_LOCALSTORE/$dailySkim

        ./prepareMerging.sh 
        numberFilesToMerge=`ls *.root|wc -l`
        echo "numberFilesToMerge: " $numberFilesToMerge
        cmsRun merge.py >& merging.log 
        echo -e "done \n"
        cd -
        ;;
    
    4e) 
        ###
        echo -e "Storing merged edm file in CASTOR..."
        mergedName=$HEDS_LABEL.root
        mergedName=merged_$mergedName
        echo "mergedName: " $mergedName
        cd $HEDS_LOCALSTORE/$dailySkim
        if ((! $?)); then
            mv merged.root ../$mergedName
            echo "going to copy the merged file into: " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            rfcp ../$mergedName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION/          
            cd -
        fi
        echo -e "done \n"
        ;;
    
    4f) 
        ###
        echo "Preparing detailed list of skimmed events..."
        cp $HEDSPATH/scripts/listSkimmedEvent.C $HEDS_LOCALSTORE            
        cd $HEDS_LOCALSTORE
        root.exe -b -q "listSkimmedEvent.C+(\"$HEDS_LABEL\")" >& /dev/null
        listSkimmedEventsPath=$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        if [ ! -d $listSkimmedEventsPath ]
        then
            mkdir $listSkimmedEventsPath
        fi
        listDetailedLogs=`ls $HEDS_LABEL*`
        for j in $listDetailedLogs
        do
            echo "preparing log " $j
            # sort entries by date
            cat $j |sort -k7 > tmp
            mv tmp $j
            mv $j $listSkimmedEventsPath
        done
        echo -e "done \n"
        cd -
        ;;
    
    4g)
        ###
        echo "Copying logs about run selection..."
        mv $HEDS_LABEL.runselection $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        mv $HEDS_LABEL.runinterval $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        echo -e "done \n"
        ;;

    4h)
        ###
        echo "Preparing event dumps..."
        input=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "input: " $input
        sed s#INPUT#$input# $HEDSPATH/scripts/dumpEvent.py > dumpEvent.py
        cmsRun dumpEvent.py >& $HEDS_LABEL.dumperLog
        $HEDSPATH/scripts/parseDumperOutput.sh 1 >& $HEDS_LABEL.dump
        $HEDSPATH/scripts/parseDumperOutput.sh 2 
        rm dumpEvent.py
        rm $HEDS_LABEL.dumperLog
        rm $HEDS_LABEL.dump
        echo -e "done \n"
        ;;
    4i)
        ###
        echo "Preparing html tables..."
        if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
            echo "create first version of FirstTable.sh"
            $HEDSPATH/scripts/editFirstTable.sh create
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
        else
            $HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
        fi
        echo -e "done \n"
        ;;

    4j) ###
        echo -e "Your preliminary table is available at the following URL: "
        url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/pagesInPreliminaryShape"
        url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
        echo -e $url "\n"
        echo -e "After you verified that all the tables have been correctly produced, " \
            "they can be moved to the final repository \n"

        ;;

    5a)
        echo ".... preparing event display server"
        mkdir $HEDS_LABEL.shots
        port=3005
        $HEDSPATH/scripts/eventDisplayServer.sh $port &
        ;;
    5b)
        echo "getting snapshots"
        port=3005
        file=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "$file" | nc -w 10 localhost $port
        ;;
    5c)
        echo "moving snapshots"
        mv $HEDS_LABEL.shots $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
        cp $HEDSPATH/scripts/indexFileForDetailedSnapShots.php \
            $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.shots/index.php
        ;;


    final)
        #TO BE FIXED. this call could written in a  much more compact way
        echo "WWW pages are going to be moved from the temporary repository"
        echo -e "("$HEDS_WWWAREA_TMP")\n"
        echo "to the permanent one"
        echo -e "("$HEDS_WWWAREA")\n"

        if [ ! -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
            mkdir $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION
        fi
        
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm.BAK
        fi
        
        mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
        mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/

        mv -T $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents


        url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/"
        url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
        
        echo "The new tables created by you are accessible through the following URL: "
        echo $url
        ;;

    #####################  after the crab part is done ###################
    99) echo "Doing bakcup of $HEDS_WWWAREA_TMP"
        date=`date +%k\.%M\.%S_%e\.%m\.%Y`
        fileName="HiggsIES.BAK."$date".tgz"
        echo "file name: " $fileName
        tar -czvf $fileName $HEDS_WWWAREA_TMP
        mv $fileName $HEDS_WWWAREA_TMP/..
        ;;


    *)
        echo "option not defined"
        ;;
esac
Revision:	1.3
Committed:	Fri May 7 13:24:44 2010 UTC (14 years, 11 months ago) by mangano
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	V00-00-00
Changes since 1.2:	+118 -27 lines
Log Message:	made scripts more robust and improve print-out
#	User	Rev	Content
1	mangano	1.1	#!/bin/bash
2
3			######################################################
4			# heds 0 (create new directories)
5			# heds 1 (query runregistry)
6			# heds 2 (create crab task)
7			# heds 3 (get and publish jobs)
8			# heds 4 (create all the html tables and store logs)
9			# heds 5* (event display stuff)
10			######################################################
11
12			choice=$1
13
14			dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT
15
16
17
18			case $choice in
19			0)
20			echo "=== you selected option 0. Setting up directories and one-time things ==="
21			echo ""
22
23	mangano	1.3	### create local folder
24	mangano	1.1	if [ -d $HEDS_LOCALSTORE ]
25	mangano	1.3	then echo "directory " $HEDS_LOCALSTORE "already exists. Then skipping \"mkdir\" "
26	mangano	1.1	else
27			echo "making " $HEDS_LOCALSTORE " directory"
28			mkdir -p $HEDS_LOCALSTORE
29			fi
30
31	mangano	1.3	### create WWW folders
32	mangano	1.2	if [ -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION ]
33	mangano	1.3	then echo "directory " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION \
34			" already exists. Then skipping \"mkdir\" "
35
36			if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
37			echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
38			mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
39			fi
40
41	mangano	1.1	else
42	mangano	1.2	echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION " directory"
43	mangano	1.3	mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
44
45			if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
46			echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA."
47			echo "Copying base html tables from there."
48			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
49			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
50			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
51			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
52			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
53			$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
54			fi
55
56			if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
57			echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
58			mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
59			fi
60	mangano	1.1	fi
61	mangano	1.3
62
63			### create CASTOR folder
64	mangano	1.1	nsls -d $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION >& tmp
65			check=`cat tmp \|grep "No such" \|wc -w`
66			rm tmp
67			if [ $check == 0 ]
68			then echo "directory " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION "already exists"
69			else
70			echo "making " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION " directory"
71			rfmkdir -p $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
72			fi
73			;;
74
75			1)
76			echo "=== you selected option 1. Querying RunRegistry, select good runs and submit crab jobs ==="
77			echo ""
78
79			dateForDbsSearch=`echo $HEDS_LABEL\|awk -F\. '{print $3"-"$2"-"$1}'`
80			echo "dateForDbsSearch: " $dateForDbsSearch
81			dbs search --query="find run where dataset=$HEDS_INPUTDATASET and run.createdate=$dateForDbsSearch" \
82			> tmpRunList.txt
83			lastRun=`cat tmpRunList.txt \|grep [0-9][0-9][0-9][0-9][0-9][0-9] \|head -n1`
84			firstRun=`cat tmpRunList.txt\|tail -n1`
85			rm tmpRunList.txt
86			#TO BE FIXED: add a protection for when there are no runs
87			echo "first,last runs: " $firstRun $lastRun
88
89
90			runselection="runselection="
91			cat $HEDSPATH/scripts/runreg.cfg \| sed s/SET_RUNMIN/$firstRun/ \| \
92			sed s/SET_RUNMAX/$lastRun/ > runreg.cfg
93			python $HEDSPATH/scripts/runregparse.py > tmp.log
94			rm runreg.cfg
95			echo "here we have the good run list"
96
97			IFS=$'\n'
98			found=0
99			lastGoodRun=0
100			for line in $(cat tmp.log);
101			do
102			if [ $found == 0 ]
103			then
104			found=`echo $line \|grep lumisToProcess\|wc \|awk '{print $1}'`
105			continue;
106			fi
107			if [ $line == ")" ]
108			then
109			break;
110			fi
111			thisRun=`echo $line \| sed s/\'//g \| sed s/,//g \|awk -F: '{print $1}'`
112			if [ $thisRun != $lastGoodRun ]
113			then
114			runselection=$runselection$thisRun\,
115			fi
116			lastGoodRun=$thisRun
117			done
118			#echo "========= THE FOLLOWING LINE TO YOUR crab.cfg FILE ========="
119			echo $runselection
120			#echo "================================================================"
121			echo $runselection > $HEDS_LABEL.runselection
122			echo $firstRun-$lastRun > $HEDS_LABEL.runinterval
123			rm tmp.log
124
125			check=`echo $runselection \| sed s/runselection=// \|wc -w`
126
127			if [ $check == 0 ]
128			then
129			echo "No good runs have been selected. No CRAB cfg will be created."
130			else
131			echo "Some good runs have been selected. CRAB cfg will be created."
132			cat $HEDSPATH/scripts/crab.template.cfg \|sed "s#SET_DATASET#$HEDS_INPUTDATASET#" \| \
133	mangano	1.3	# sed "s#SET_REMOTEDIR#$HEDS_USERID/HEDS/#" \| \
134			sed "s#SET_REMOTEDIR#$HEDS_USERID/#" \| \
135	mangano	1.1	sed "s#SET_T2#$HEDS_T2SITE#" \| \
136			sed "s/SET_RUNSELECTION/$runselection/" > tmp.cfg
137			pset=$HEDSPATH/python/earlyDataInterestingEvents_cfg.py
138			echo "pset: " $pset
139			cat tmp.cfg \|sed s#SET_PSET#$pset# \| sed "s#SET_TASK_NAME#$dailySkim#" > tmp2.cfg; rm tmp.cfg
140			mv tmp2.cfg $HEDS_LABEL.crab.cfg
141			fi
142			;;
143			2)
144			echo "creating and submitting CRAB jobs for " $dailySkim
145			if [ -e $HEDS_LABEL.crab.cfg ]
146			then
147			crab -create -submit -cfg $HEDS_LABEL.crab.cfg
148			if ((! $?)); then rm $HEDS_LABEL.crab.cfg; fi
149			else
150			echo "ERROR: " $HEDS_LABEL.crab.cfg " doesn't exist"
151			fi
152			;;
153			3)
154			echo "getting output and publishing crab jobs"
155			crab -status -c $dailySkim
156			crab -get -c $dailySkim
157			crab -status -c $dailySkim
158			crab -publish -c $dailySkim
159			;;
160
161
162			4)
163			echo "option 4 "
164			if [ -d $dailySkim ] #the crab job was run on at least 1 run
165			then
166			heds 4a
167			heds 4b
168			heds 4c
169			heds 4d
170			heds 4e
171			heds 4f
172			heds 4g
173			heds 4h
174			heds 4i
175			else
176			echo "copying only the log files about runs and selected-runs"
177			heds 4g
178			fi
179			;;
180
181			4a)
182	mangano	1.3	#### echo "...copying log files"
183	mangano	1.1	if [ ! -d $HEDS_LOCALSTORE/$dailySkim/logs ]; then
184			mkdir -p $HEDS_LOCALSTORE/$dailySkim/logs
185			fi
186			cp $dailySkim/res/.std $HEDS_LOCALSTORE/$dailySkim/logs
187			if (($?)); then
188			echo "ERROR: failed to copy log files in " $HEDS_LOCALSTORE/$dailySkim/logs;
189			else
190			echo "copied all logs file in " $HEDS_LOCALSTORE/$dailySkim/logs
191			fi
192
193			###
194			tarName=$HEDS_LABEL.log.tgz
195			cd $HEDS_LOCALSTORE/$dailySkim/logs/
196			if (($?));then
197			echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
198			else
199	mangano	1.3	echo "compriming log files... "
200			tar czvf $tarName .std >& /dev/null
201			echo -e "done \n"
202
203			echo "copying " $tarName " file to CASTOR area.."
204	mangano	1.1	rfcp $tarName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
205	mangano	1.3	echo -e "done \n"
206	mangano	1.1	cd -
207			fi
208			;;
209
210			4b)
211			###
212	mangano	1.3	echo -e "creating skimReport..."
213	mangano	1.1	logName=$HEDS_LABEL.log
214			$HEDSPATH/scripts/skimreport $HEDS_LOCALSTORE/$dailySkim/logs/*.stdout > $logName
215	mangano	1.3	mv $logName $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents/
216			echo -e "done \n"
217	mangano	1.1	;;
218
219			4c)
220			###
221			echo "...coping files from the T2's SE"
222			echo "dataset to retreive: " $dailySkim
223			dbs search --url=https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet \
224			--query="find file,site where dataset=$dailySkim" \|grep .root >listFiles
225
226			IFS=$'\n'
227			for j in $(cat listFiles)
228			do
229			echo "going to copy file: $j"
230			fileName=`echo $j \|awk '{print$1}'\| awk -F$dailySkim/ '{print $2}' \| awk -F/ '{print $2}'`
231
232			storage_elem=`echo $j \|awk '{print $2}'`
233			remote_path=`echo $j\|awk '{print $1}'`
234			lcg-cp -b -D srmv2 srm://$storage_elem:$HEDS_STORAGEPORT$HEDS_STORAGEPATH$remote_path file://$HEDS_LOCALSTORE/$dailySkim/$fileName
235			done
236			rm listFiles
237			nFiles=`ls $HEDS_LOCALSTORE/$dailySkim/*.root\|wc -l`
238			nJobs=`ls $HEDS_LOCALSTORE/$dailySkim/logs/CMSSW_*.stdout \|wc -l`
239
240			### TO BE FIXED!! this check isn't valid if some job selects zero events (no file is created)
241			#if [ $nFiles == $nJobs ]
242			#then
243			# echo "GOOD: #files copied from hadoop matches #jobs in the CRAB task"
244			#else
245			# echo "ERROR: #files in hadoop doesn't match #jobs in the CRAB task"
246			# exit
247			#fi
248			;;
249
250			4d)
251			###
252	mangano	1.3	echo -e "Merging files edm files..."
253	mangano	1.1	cp $HEDSPATH/scripts/prepareMerging.sh $HEDS_LOCALSTORE/$dailySkim
254			cp $HEDSPATH/scripts/merge.py $HEDS_LOCALSTORE/$dailySkim
255			cd $HEDS_LOCALSTORE/$dailySkim
256
257			./prepareMerging.sh
258			numberFilesToMerge=`ls *.root\|wc -l`
259			echo "numberFilesToMerge: " $numberFilesToMerge
260			cmsRun merge.py >& merging.log
261	mangano	1.3	echo -e "done \n"
262	mangano	1.1	cd -
263			;;
264
265			4e)
266			###
267	mangano	1.3	echo -e "Storing merged edm file in CASTOR..."
268	mangano	1.1	mergedName=$HEDS_LABEL.root
269			mergedName=merged_$mergedName
270			echo "mergedName: " $mergedName
271			cd $HEDS_LOCALSTORE/$dailySkim
272			if ((! $?)); then
273			mv merged.root ../$mergedName
274			echo "going to copy the merged file into: " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
275			rfcp ../$mergedName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION/
276			cd -
277			fi
278	mangano	1.3	echo -e "done \n"
279	mangano	1.1	;;
280
281			4f)
282			###
283	mangano	1.3	echo "Preparing detailed list of skimmed events..."
284	mangano	1.1	cp $HEDSPATH/scripts/listSkimmedEvent.C $HEDS_LOCALSTORE
285			cd $HEDS_LOCALSTORE
286	mangano	1.3	root.exe -b -q "listSkimmedEvent.C+(\"$HEDS_LABEL\")" >& /dev/null
287	mangano	1.2	listSkimmedEventsPath=$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
288	mangano	1.1	if [ ! -d $listSkimmedEventsPath ]
289			then
290			mkdir $listSkimmedEventsPath
291			fi
292			listDetailedLogs=`ls $HEDS_LABEL*`
293			for j in $listDetailedLogs
294			do
295			echo "preparing log " $j
296			# sort entries by date
297			cat $j \|sort -k7 > tmp
298			mv tmp $j
299			mv $j $listSkimmedEventsPath
300			done
301	mangano	1.3	echo -e "done \n"
302	mangano	1.1	cd -
303			;;
304
305			4g)
306			###
307	mangano	1.3	echo "Copying logs about run selection..."
308			mv $HEDS_LABEL.runselection $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
309			mv $HEDS_LABEL.runinterval $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
310			echo -e "done \n"
311	mangano	1.1	;;
312
313			4h)
314			###
315	mangano	1.3	echo "Preparing event dumps..."
316	mangano	1.1	input=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
317			echo "input: " $input
318			sed s#INPUT#$input# $HEDSPATH/scripts/dumpEvent.py > dumpEvent.py
319			cmsRun dumpEvent.py >& $HEDS_LABEL.dumperLog
320			$HEDSPATH/scripts/parseDumperOutput.sh 1 >& $HEDS_LABEL.dump
321			$HEDSPATH/scripts/parseDumperOutput.sh 2
322			rm dumpEvent.py
323			rm $HEDS_LABEL.dumperLog
324			rm $HEDS_LABEL.dump
325	mangano	1.3	echo -e "done \n"
326	mangano	1.1	;;
327			4i)
328			###
329	mangano	1.3	echo "Preparing html tables..."
330	mangano	1.2	if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
331	mangano	1.1	echo "create first version of FirstTable.sh"
332			$HEDSPATH/scripts/editFirstTable.sh create
333	mangano	1.3	$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
334			$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
335			$HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
336			else
337			$HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
338			$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
339			$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
340			$HEDSPATH/scripts/editFirstTable.sh add below ; echo "level-1 table modified"
341			fi
342			echo -e "done \n"
343			;;
344
345			4j) ###
346			echo -e "Your preliminary table is available at the following URL: "
347			url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/pagesInPreliminaryShape"
348			url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
349			echo -e $url "\n"
350			echo -e "After you verified that all the tables have been correctly produced, " \
351			"they can be moved to the final repository \n"
352
353	mangano	1.1	;;
354	mangano	1.3
355	mangano	1.1	5a)
356			echo ".... preparing event display server"
357			mkdir $HEDS_LABEL.shots
358			port=3005
359			$HEDSPATH/scripts/eventDisplayServer.sh $port &
360			;;
361			5b)
362			echo "getting snapshots"
363			port=3005
364			file=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
365			echo "$file" \| nc -w 10 localhost $port
366			;;
367			5c)
368			echo "moving snapshots"
369	mangano	1.2	mv $HEDS_LABEL.shots $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
370	mangano	1.1	cp $HEDSPATH/scripts/indexFileForDetailedSnapShots.php \
371	mangano	1.2	$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.shots/index.php
372	mangano	1.1	;;
373
374
375	mangano	1.3	final)
376			#TO BE FIXED. this call could written in a much more compact way
377			echo "WWW pages are going to be moved from the temporary repository"
378			echo -e "("$HEDS_WWWAREA_TMP")\n"
379			echo "to the permanent one"
380			echo -e "("$HEDS_WWWAREA")\n"
381
382			if [ ! -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
383			mkdir $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION
384			fi
385
386			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ]; then
387			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
388			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php.BAK
389			fi
390			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm ]; then
391			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
392			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm.BAK
393			fi
394			if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.php ]; then
395			cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
396			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm.BAK
397			fi
398
399			mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.php \
400			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
401			mv $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.shtm \
402			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
403
404			mv -T $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents \
405			$HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
406
407
408			url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS/"
409			url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
410
411			echo "The new tables created by you are accessible through the following URL: "
412			echo $url
413			;;
414
415	mangano	1.1	##################### after the crab part is done ###################
416	mangano	1.2	99) echo "Doing bakcup of $HEDS_WWWAREA_TMP"
417	mangano	1.1	date=`date +%k\.%M\.%S_%e\.%m\.%Y`
418			fileName="HiggsIES.BAK."$date".tgz"
419			echo "file name: " $fileName
420	mangano	1.2	tar -czvf $fileName $HEDS_WWWAREA_TMP
421			mv $fileName $HEDS_WWWAREA_TMP/..
422	mangano	1.1	;;
423
424
425			*)
426			echo "option not defined"
427			;;
428			esac