EarlyDataStudy/scripts/heds.sh

#!/bin/bash

######################################################
# heds 0 (create new directories)
# heds 1 (query runregistry)
# heds 2 (create crab task)
# heds 3 (get and publish jobs)
# heds 4 (create all the html tables and store logs)
# heds 5* (event display stuff)
######################################################

choice=$1

dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT


case $choice in
######### "public functions" which are meant to be called by the users ################  
    up) ### bootstrap for heds job submission. It creates the intial configuration file
        whoami=`whoami`
        cat > ./heds.cfg <<EOF
### insert a day of data-taking in the format DD.MM.YYYY
HEDS_LABEL=18.04.2010

### name of the local folder containing the configuration and output of your heds task
HEDS_LOCALSTORE=""

### input sample
#HEDS_INPUTDATASET=/MinimumBias/Commissioning10-Apr20ReReco-v1/RECO #up to run 133532
HEDS_INPUTDATASET=/MinimumBias/Commissioning10-PromptReco-v9/RECO #starting from run 133532


HEDS_SKIMVERSION=99 
HEDS_TASKATTEMPT=1

HEDS_USERID=$whoami
HEDS_T2SITE=T2_US_UCSD


### TO BE FIXED. THESE VARIABLE COULD BE RETRIEVED FROM A QUERY TO PHEDEX ###
HEDS_STORAGEPATH=/srm/v2/server?SFN=/hadoop/cms  
HEDS_STORAGEPORT=8443
EOF
        echo -e "+++ Bootstrap of \"heds\" scripts peformed"
        echo -e "+++ The configuration file \"heds.cfg\" has been created in the current folder:"
        echo -e "    "$PWD "\n"
        echo "+++ Edit the heds.cfg according to your needs and then execute: "
        echo -e "    ""heds.sh create heds.cfg"
        ;;

    create) ### create local heds taskFolder
        configFile=$2
        if [ -z "$configFile" ]; then
            echo "+++ ERROR: you have to specify the configuration file as in:"
            echo "    heds.sh create <heds.cfg>"
            exit 1
        fi

        if [ -z "$CMSSW_BASE" ]; then
            echo "+++ ERROR: you have to setup CMSSW environment first. Run cmsenv";
            (exit 1;)
        fi

        if [ ! -d $CMSSW_BASE/src/HiggsAnalysis/EarlyDataStudy/scripts ]; then
            echo "+++ ERROR: you have to checkout and compile \"HiggsAnalysis/EarlyDataStudy\" first.";
            (exit 1;)
        fi


        # get rid of lines with comments
        grep -v '^#' $configFile > tmpFile
        mv tmpFile ${configFile} 

        # add export lines
        grep -v '^$' ${configFile}| awk '{print "export "$0}'  > tmpFile
        mv tmpFile ${configFile}

        # define default taskFolder if it is not defined in the cfg file
        autoString=heds_taskFolder_`date '+%d_%m_%Y_%H_%M'`
        source ${configFile}
        if [ -z "$HEDS_LOCALSTORE" ]; then
            sed "s#HEDS_LOCALSTORE=\"\"#HEDS_LOCALSTORE=$autoString#" ${configFile} > tmpFile
            mv tmpFile ${configFile}
        fi

        # make taskFolder path absolute
        tmpString="HEDS_LOCALSTORE=`pwd`/"
        sed "s#HEDS_LOCALSTORE=#$tmpString#"  ${configFile} > tmpFile
        mv tmpFile ${configFile}

        source ${configFile}

        cat >> ${configFile} <<EOF
###### this part was not set by the user. It was automatically generated ########
export HEDS_NAMEPREFIX=HiggsSimpleSkimV${HEDS_SKIMVERSION}\_
export HEDS_CASTORFOLDER=/castor/cern.ch/user/${HEDS_USERID:0:1}/$HEDS_USERID/HEDS 
export HEDS_WWWAREA_TMP=${HEDS_LOCALSTORE}/wwwPagesInPreliminaryShape
export HEDS_WWWAREA=/afs/cern.ch/cms/Physics/Higgs/HiggsDQM/HiggsEDS
export HEDSPATH=$CMSSW_BASE/src/HiggsAnalysis/EarlyDataStudy
EOF


        if [ -d $HEDS_LOCALSTORE ]; then
            echo "ERROR: the folder" $HEDS_LOCALSTORE "already exists." \
                "Plese specify a different one in your .cfg file"
            exit 1
        else
            mkdir $HEDS_LOCALSTORE
            mv ./${configFile} $HEDS_LOCALSTORE/configuration
        fi

        cat > $HEDS_LOCALSTORE/status <<EOF
create
EOF


        ;;

    submit) ### start doing something serious using the settings stored in the hedsTaskFolder
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh submit <hedsTaskFolder>"
            exit 1
        fi
        
        echo -e "+++ Your heds task is going to use the following settings: \n"
        
        source ./$hedsTaskFolder/configuration
        env |grep -i heds
        echo -e "\n"

        echo -e "+++ Are you Ok with such settings? [y/n] "


        read -p "$prompt" answer
        case "$answer" in
            [yY1] )
                echo -e "\n"
                ;;
            [nN0] )
                exit
                ;;
            *) 
                echo -e "Incorrect answer. Run \"heds submit\" again and type [y/n]"
                exit
                ;;
        esac

        echo -e "+++ Have you also checked HEDS_STORAGEPATH and HEDS_STORAGEPORT values ? "
        echo -e "+++ This values have still to be set by hand. Please consult the settings of your T2's SE"
        echo -e "    as described here:"
        echo    "    https://twiki.cern.ch/twiki/bin/view/CMS/AnalysisOpsPriorityuser#Tier_2_Sites_Responsibility"
        echo    "Please insert the final confirmation [y/n]"
        read -p "$prompt" answer
        case "$answer" in
            [yY1] )
                echo "you said you are happy"
                ;;
            [nN0] )
                exit
                ;;
            *) 
                echo -e "Incorrect answer. Run \"heds submit\" again and type [y/n]"
                exit
                ;;
        esac


        initialFolder=`pwd`
        cd $HEDS_LOCALSTORE
        heds.sh 0 #create setup
        heds.sh 1 #query runregistry, create crab cfg
        heds.sh 2 #submit crab jobs

        cd $initialFolder
        ;;


    status) ### check status
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh status <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration

        initialFolder=`pwd`
        cd $HEDS_LOCALSTORE
        status=`tail -n1 status`

        case $status in
            create)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds create <taskFolder>\n"
                echo "+++ Next step to run is: "
                echo -e "    heds submit <taskFolder>"          
                ;;
            submit)
                echo "+++ Last step execute was: "
                echo -e "    heds submit <taskFolder>\n"

                dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT
                crab -status -c $HEDS_LOCALSTORE/$dailySkim >& crab.status.log
                jobsOK=`grep "Wrapper Exit Code : 0" crab.status.log| awk '{print $2}'`
                jobsDone=`grep ">>>>>>>>>" crab.status.log |grep "Jobs Done"| awk '{print $2}'`
                jobsRunning=`grep ">>>>>>>>>" crab.status.log |grep "Jobs Running"| awk '{print $2}'`
                jobsAborted=`grep ">>>>>>>>>" crab.status.log |grep "Jobs Aborted"| awk '{print $2}'`
                jobsTotal=`grep "crab:" crab.status.log |grep "Total Jobs"| awk '{print $2}'`
                
                if [ -n "$jobsTotal" ];   then echo "# jobs Total:        " $jobsTotal;   else jobsTotal=0; fi
                if [ -n "$jobsDone" ];    then echo "# jobs Done:         " $jobsDone;    else jobsDone=0; fi
                if [ -n "$jobsRunning" ]; then echo "# jobs Running:      " $jobsRunning; else jobsRunning=0; fi
                if [ -n "$jobsAborted" ]; then echo "# jobs Aborted:      " $jobsAborted; else jobsAborted=0; fi
                if [ -n "$jobsOK" ];      then echo "# jobs Retrieved OK: " $jobsOK;      else jobsOK=0; fi

                if(($jobsDone)); then
                    echo "There are jobs in DONE status. Retrieving them"
                    crab -get -c $HEDS_LOCALSTORE/$dailySkim > /dev/null
                    
                    echo "+++ Run the following command again: "
                    echo "    heds status <taskFolder>"
                    exit 0
                fi

                if [ "$jobsTotal" == "$jobsOK" ];
                then
                    echo "+++ all the crab jobs has been correctly retrieved."          
                    echo "+++ You can now run:"
                    echo "    heds get <taskFolder>"
                    cat >> $HEDS_LOCALSTORE/status <<EOF
crab_done
EOF
                else
                    if(($jobsAborted)); then
                        echo "+++ Some crab jobs aborted. You need to resubmit them directly using:"
                        echo "    crab -resubmit <job-interval> -c $HEDS_LOCALSTORE/$dailySkim"

                    else
                        echo "+++ There are still crab jobs running. Try again later."
                        echo "+++ You can directly handle your crab jobs with:"
                        echo "    crab -<command> -c $HEDS_LOCALSTORE/$dailySkim"
                    fi
                fi
                ;;

            submit_empty)
                echo "+++ Last step execute was: "
                echo -e "    heds submit <taskFolder>\n"
                
                echo "+++ No good runs were found, so no crab jobs were submitted."
                echo "    You can directly go to the step:"
                echo "    heds tables <taskFolder>"
                ;;

            crab_done)
                echo "+++ All you crab jobs are DONE."
                echo "    You can now run:"
                echo "    heds get <taskFolder>"
                ;;

            get)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds get <taskFolder>\n"
                echo "+++ Next step to run is: "
                echo -e "    heds merge <taskFolder>"           
                ;;

            merge)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds merge <taskFolder>\n"
                echo "+++ Next step to run is: "
                echo -e "    heds tables <taskFolder>"          
                ;;

            tables)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds tables <taskFolder>\n"
                echo "+++ Next step to run is: "
                echo -e "    heds preview <taskFolder>"         
                ;;

            preview)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds preview <taskFolder>\n"
                echo "+++ Next step to run is: "
                echo -e "    heds add2Web <taskFolder>"         
                ;;

            add2Web)
                echo "+++ Last step successfully execute was: "
                echo -e "    heds add2Web <taskFolder>\n"
                echo "+++ You are done "
                ;;
            
            *)
                echo "+++ ERROR: undefined heds status."
                echo "+++ Restart from scratch running:"
                echo "    heds up"
                echo "    heds create heds.cfg"
                ;;
        esac
        cd $initialFolder
        ;;


    get) ### get output from crab
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh status <hedsTaskFolder>"
            exit 1
        fi

        source ./$hedsTaskFolder/configuration
        cd $HEDS_LOCALSTORE
        status=`tail -n1 status`
        if [ ! "$status" == "crab_done" ];then
            echo "+++ ERROR: crab jobs are not done yet. Try:"
            echo "    heds status <taskFolder>"
            exit 1
        fi

        echo "Crab job's output is going to be published in DBS. This step is going to take a long while (up to 1hr)."
        echo "Do you have time for that? [y/n]"
        
        read -p "$prompt" answer
        case "$answer" in
            [yY1] )
                echo "Ok. Proceding with crab -publish step"
                ;;
            [nN0] )
                echo "Exiting heds.."
                exit 0
                ;;
            *) 
                echo -e "Incorrect answer. Run your heds get command again and then answer [y/n]"
                exit
                ;;
        esac


        dailySkim=$HEDS_NAMEPREFIX$HEDS_LABEL-v$HEDS_TASKATTEMPT
        crab -publish -c $HEDS_LOCALSTORE/$dailySkim
        if((! $?)); then
            heds.sh 4c #copy file from the SE
        else
            exit 1
        fi
        #heds.sh 4c_temp #copy file from the SE manually using scp

        if ((! $?));
        then
            cat >> status <<EOF
get
EOF
        fi
        cd $initialFolder
        ;;


    tables) ### produce html tables
        hedsTaskFolder=$2
        rawLocation=$3
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh tables <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration
        cd $HEDS_LOCALSTORE
        status=`tail -n1 status`
        echo "status: " $status
        if [[ !("$status" == "merge") && !("$status" == "submit_empty")  ]];then
            echo "+++ ERROR: you may have to retrieve crab jobs first. Try:"
            echo "    heds status <taskFolder>"
            exit 1
        fi

        
        if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
            echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA." 
            echo "Copying base html tables from there."
            
                ##TO BE FIXED. Check that files exist before copy
            cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
                $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
            cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
                $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
            cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
                $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
        fi

        if [ -z "$rawLocation" ];then
            rawLocation=below
        fi

        if [ "$status" == "merge" ]; then
            heds.sh 4a #copy all logs file
            heds.sh store . #copy zipped edm file and crab logs
            heds.sh 4b #making skim report
            heds.sh 4f #list of skimmed events
            heds.sh 4g #copy logs about run selection
            heds.sh 4h #preparing event dumps
            heds.sh 4i $rawLocation #preparing final html tables
        fi

        if [ "$status" == "submit_empty" ]; then
            heds.sh 4g  #copy logs about run selection
            heds.sh 4i2 $rawLocation #preparing final html tables
        fi

        cat >> status <<EOF
tables
EOF
        cd $initialFolder
        ;;

    merge) ### merge files before copying them to castor area
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh tables <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration
        cd $HEDS_LOCALSTORE
        status=`grep "crab_done" status`
        if [ -z "$status" ];then
            echo "+++ ERROR: crab jobs are not done yet. Try:"
            echo "    heds status <taskFolder>"
            exit 1
        fi

        heds.sh 4d #merging edm files

        cat >> status <<EOF
merge
EOF
        cd $initialFolder
        ;;


    store) ### produce html tables
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh tables <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration
        cd $HEDS_LOCALSTORE
        status=`grep "crab_done" status`
        if [ -z "$status" ];then
            echo "+++ ERROR: crab jobs are not done yet. Try:"
            echo "    heds status <taskFolder>"
            exit 1
        fi

        ### create CASTOR folder
        nsls -d $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION >& tmp
        check=`cat tmp |grep "No such" |wc -w`
        rm tmp
        if [ $check == 0 ]
        then echo "directory " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION "already exists"           
        else
            echo "making " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION " directory"
            rfmkdir -p $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION            
        fi

        heds.sh 6 #copy merged log files in CASTOR
        heds.sh 4e #copy merged edm files in CASTOR
#       cat >> status <<EOF
#store
#EOF
        cd $initialFolder
        ;;

    preview) ### produce html tables
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh tables <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration

        initialFolder=`pwd`
        cd $HEDS_LOCALSTORE
        heds.sh 4j #
        php $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php > FirstPage.html
        mv FirstPage.html $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
        cp $HEDS_WWWAREA/fromGiovanni.css $HEDS_WWWAREA_TMP
        cat >> status <<EOF
preview
EOF
        cd $initialFolder
        ;;

    add2Web) ### produce html tables
        hedsTaskFolder=$2
        if [ -z "$hedsTaskFolder" -o ! -d ./$hedsTaskFolder ]; then
            echo -e "+++ ERROR: you have to specify a \"hedsTaskFolder\" as in: "
            echo "    heds.sh tables <hedsTaskFolder>"
            exit 1
        fi
        source ./$hedsTaskFolder/configuration

        initialFolder=`pwd`
        cd $HEDS_LOCALSTORE
        heds.sh final #
        cat >> status <<EOF
add2Web
EOF
        cd $initialFolder
        ;;
    
    
########## internal "functions" which are not meant to be used by users ###############
    0)
        echo "=== Setting up directories and one-time things ==="
        echo ""

        ### create WWW folders
        if [ -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION ]
        then echo "directory " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION \
            " already exists. Then skipping \"mkdir\" "

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi

        else
            echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION " directory"
            mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/       

            if [ -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
                echo "Warning: " logs.SkimV$HEDS_SKIMVERSION " already exist in final WWW_AREA." 
                echo "Copying base html tables from there."

                ##TO BE FIXED. Check that files exist before copy
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
                cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
                    $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
            fi

            if [ ! -d $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "making " $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
                mkdir -p $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi
        fi
        ;;

    1)
        echo "=== you selected option 1. Querying RunRegistry, select good runs and submit crab jobs ==="
        echo ""

        dateForDbsSearch=`echo $HEDS_LABEL|awk -F\. '{print $3"-"$2"-"$1}'`
        #echo "dateForDbsSearch: " $dateForDbsSearch
        dbs search --query="find run where dataset=$HEDS_INPUTDATASET and run.createdate=$dateForDbsSearch" \
            > tmpRunList.txt    
        lastRun=`cat tmpRunList.txt |grep [0-9][0-9][0-9][0-9][0-9][0-9] |head -n1`
        firstRun=`cat tmpRunList.txt|grep [0-9][0-9][0-9][0-9][0-9][0-9] |tail -n1`


        if((!${#lastRun}));then
            rm tmpRunList.txt
            echo "no runs" > $HEDS_LABEL.runinterval

            echo "No input runs for"
            echo "HEDS_LABEL=" $HEDS_LABEL 
            echo "and"
            echo "HEDS_INPUTDATASET=" $HEDS_INPUTDATASET
            echo ""

            echo "No CRAB cfg will be created."
            cat >> status <<EOF
runselection_empty
EOF
            exit 0
        fi

        rm tmpRunList.txt
        echo "first,last runs: " $firstRun $lastRun


        runselection="runselection="
        cat $HEDSPATH/scripts/runreg.cfg | sed s/SET_RUNMIN/$firstRun/ | \
            sed s/SET_RUNMAX/$lastRun/ > runreg.cfg
        python $HEDSPATH/scripts/runregparse.py  > tmp.log
        rm runreg.cfg
        echo "here we have the good run list"

        IFS=$'\n'
        found=0
        lastGoodRun=0
        for line in $(cat tmp.log);
        do
            if [ $found == 0 ]
            then
                found=`echo $line |grep lumisToProcess|wc |awk '{print $1}'`
                continue;
            fi
            if [ $line == ")" ] 
            then  
                break;
            fi
            thisRun=`echo $line | sed s/\'//g | sed s/,//g |awk -F: '{print $1}'`
            if [ $thisRun != $lastGoodRun ] 
            then
                runselection=$runselection$thisRun\,
            fi
            lastGoodRun=$thisRun
        done
        #echo "========= THE FOLLOWING LINE TO YOUR crab.cfg FILE ========="
        echo $runselection
        #echo "================================================================"
        echo $runselection > $HEDS_LABEL.runselection
        echo $firstRun-$lastRun > $HEDS_LABEL.runinterval
        rm tmp.log

        check=`echo $runselection | sed s/runselection=// |wc -w`

        if [ $check == 0 ] 
        then
            echo "No good runs have been selected. No CRAB cfg will be created."
            cat >> status <<EOF
runselection_empty
EOF
        else
            echo "Some good runs have been selected. CRAB cfg will be created."
            cat >> status <<EOF
runselection_done
EOF
            cat $HEDSPATH/scripts/crab.template.cfg |sed "s#SET_DATASET#$HEDS_INPUTDATASET#" | \
#               sed "s#SET_REMOTEDIR#$HEDS_USERID/HEDS/#" | \
                sed "s#SET_REMOTEDIR#$HEDS_USERID/#" | \
                sed "s#SET_T2#$HEDS_T2SITE#" | \
                sed "s/SET_RUNSELECTION/$runselection/" > tmp.cfg            
            pset=$HEDSPATH/python/earlyDataInterestingEvents_cfg.py
            echo "pset: " $pset
            cat tmp.cfg |sed s#SET_PSET#$pset# | sed "s#SET_TASK_NAME#$dailySkim#" > tmp2.cfg; rm tmp.cfg
            mv tmp2.cfg $HEDS_LABEL.crab.cfg
        fi
        ;;
    2)
        status=`tail -n1 status`
        if [ ! "$status" == "runselection_done" ];then
            #no crab jobs to be submitted
            cat >> status <<EOF
submit_empty
EOF
            exit 0
        fi

        echo "creating and submitting CRAB jobs for " $dailySkim
        if [ -e $HEDS_LABEL.crab.cfg ] 
        then 
            crab -create -submit -cfg $HEDS_LABEL.crab.cfg
            if ((! $?)); then rm $HEDS_LABEL.crab.cfg; fi
            cat >> status <<EOF
submit
EOF
        else
            echo "ERROR: " $HEDS_LABEL.crab.cfg " doesn't exist"
        fi
        ;;
    3)
        echo "getting output and publishing crab jobs"
        crab -status -c $dailySkim &> /dev/null
        crab -get -c $dailySkim 
        crab -status -c $dailySkim &> /dev/null
        crab -publish -c $dailySkim 
        ;;
    

    4)  
        echo "option 4 "
        if [ -d $dailySkim ] #the crab job was run on at least 1 run
        then
            heds 4a 
            heds 4b 
            heds 4c 
            heds 4d 
            heds 4e 
            heds 4f 
            heds 4g 
            heds 4h 
            heds 4i 
        else
            echo "copying only the log files about runs and selected-runs"
            heds 4g
            heds 4i2
        fi
        ;;
    
    4a) 
        #### echo "...copying log files"
        if [ ! -d $HEDS_LOCALSTORE/$dailySkim/logs ]; then
            mkdir -p $HEDS_LOCALSTORE/$dailySkim/logs
        fi
        cp $dailySkim/res/*.std* $HEDS_LOCALSTORE/$dailySkim/logs
        if (($?)); then
            echo "ERROR: failed to copy log files in " $HEDS_LOCALSTORE/$dailySkim/logs;
        else
            echo "copied all logs file in " $HEDS_LOCALSTORE/$dailySkim/logs
        fi

        ###
        tarName=$HEDS_LABEL.log.tgz
        cd $HEDS_LOCALSTORE/$dailySkim/logs/
        if (($?));then
            echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
        else
            echo "compriming log files... "
            tar czvf $tarName *.std* >& /dev/null
            echo -e "done \n"
            cd -
        fi
        ;;
    
    4b) 
        ###
        echo -e "creating skimReport..."
        logName=$HEDS_LABEL.log
        $HEDSPATH/scripts/skimreport $HEDS_LOCALSTORE/$dailySkim/logs/*.stdout  > $logName
        cp $logName $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents/            
        echo -e "done \n"
        ;;
        
    4c) 
        ###
        echo "...coping files from the T2's SE"
        echo "dataset to retreive: " $dailySkim
        dbs search --url=https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet \
            --query="find file,site where dataset=*$dailySkim*" |grep .root >listFiles

        IFS=$'\n'
        for j in $(cat listFiles)
        do
            echo "going to copy file: $j"
            fileName=`echo $j |awk '{print$1}'| awk -F$dailySkim/ '{print $2}' | awk -F/ '{print $2}'`

            storage_elem=`echo $j |awk '{print $2}'`
            remote_path=`echo $j|awk '{print $1}'`
            lcg-cp -b -D srmv2  srm://$storage_elem:$HEDS_STORAGEPORT$HEDS_STORAGEPATH$remote_path file://$HEDS_LOCALSTORE/$dailySkim/$fileName
        done
        rm listFiles
        #nFiles=`ls $HEDS_LOCALSTORE/$dailySkim/*.root|wc -l`
        #nJobs=`ls $HEDS_LOCALSTORE/$dailySkim/logs/CMSSW_*.stdout |wc -l`
        ;;

    4c_temp)
        echo "copying manually files from SE.. "
        scp bmangano@uaf-6.t2.ucsd.edu:/hadoop/cms/store/user/mangano/MinimumBias/$dailySkim/*/*.root $HEDS_LOCALSTORE/$dailySkim/
        ;;


    4d)
        ###
        echo -e "Merging files edm files..."
        cp $HEDSPATH/scripts/prepareMerging.sh $HEDS_LOCALSTORE/$dailySkim
        cp $HEDSPATH/scripts/merge.py $HEDS_LOCALSTORE/$dailySkim
        cd $HEDS_LOCALSTORE/$dailySkim

        mergedName=$HEDS_LABEL.root
        mergedName=merged_$mergedName

        ./prepareMerging.sh 
        numberFilesToMerge=`ls *.root|wc -l`
        echo "numberFilesToMerge: " $numberFilesToMerge
        cmsRun merge.py >& merging.log 
        mv merged.root $HEDS_LOCALSTORE/$mergedName
        echo -e "done \n"
        cd -
        ;;
    
    4e) 
        ###
        echo -e "Storing merged edm file in CASTOR..."
        mergedName=$HEDS_LABEL.root
        mergedName=merged_$mergedName
        echo "mergedName: " $mergedName
        cd $HEDS_LOCALSTORE/$dailySkim
        if ((! $?)); then
            echo "going to copy the merged file into: " $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            rfcp $HEDS_LOCALSTORE/$mergedName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION/            
            cd -
        fi
        echo -e "done \n"
        ;;
    
    4f) 
        ###
        echo "Preparing detailed list of skimmed events..."
        cp $HEDSPATH/scripts/listSkimmedEvent.C $HEDS_LOCALSTORE            
        cd $HEDS_LOCALSTORE
        forRootExecution="listSkimmedEvent.C++(\"${HEDS_LABEL}\")"

        root.exe -b -q $forRootExecution >& /dev/null
        listSkimmedEventsPath=$HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        if [ ! -d $listSkimmedEventsPath ]
        then
            mkdir $listSkimmedEventsPath
        fi
        listDetailedLogs=`ls ${HEDS_LABEL}_Skim_*`
        for j in $listDetailedLogs
        do
            echo "preparing log " $j
            # sort entries by date
            cat $j |sort -k7 > tmp
            mv tmp $j
            mv $j $listSkimmedEventsPath
        done
        echo -e "done \n"
        cd -
        ;;
    
    4g)
        ###
        echo "Copying logs about run selection..."
        cp $HEDS_LABEL.runselection $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        cp $HEDS_LABEL.runinterval $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        echo -e "done \n"
        ;;

    4h)
        ###
        echo "Preparing event dumps..."
        input=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "input: " $input
        sed s#INPUT#$input# $HEDSPATH/scripts/dumpEvent.py > dumpEvent.py
        cmsRun dumpEvent.py >& $HEDS_LABEL.dumperLog

        $HEDSPATH/scripts/parseDumperOutput.sh 1 >& $HEDS_LABEL.dump
        $HEDSPATH/scripts/parseDumperOutput.sh 2 
        rm dumpEvent.py
        rm $HEDS_LABEL.dumperLog
        rm $HEDS_LABEL.dump
        echo -e "done \n"
        ;;
    4i)
        ###
        rawLocation=$2
        echo "Preparing html tables..."
        if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
            echo "create first version of FirstTable.sh"
            $HEDSPATH/scripts/editFirstTable.sh create
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add $rawLocation ; echo "level-1 table modified"
        else
            $HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
            $HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            $HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh add $rawLocation ; echo "level-1 table modified"
        fi
        echo -e "done \n"
        ;;

    4i2)
        ###
        rawLocation=$2
        echo "Preparing html tables..."
        if [ ! -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ];then
            echo "create first version of FirstTable.sh"
            $HEDSPATH/scripts/editFirstTable.sh create
            #$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            #$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh addSpecial $rawLocation ; echo "level-1 table modified"
        else
            #$HEDSPATH/scripts/createTableSkimDetails.sh clean; echo "clean up old html tables"
            #$HEDSPATH/scripts/createTableSkimDetails.sh add ; echo "level-3 tables produced"
            #$HEDSPATH/scripts/createTableSkimSummary.sh ; echo "level-2 tables produced"
            $HEDSPATH/scripts/editFirstTable.sh addSpecial $rawLocation ; echo "level-1 table modified"
        fi
        echo -e "done \n"
        ;;


    4j) ###
        echo -e "Your preliminary table is available at the following URL: "
        url="file://"
        url=$url$HEDS_WWWAREA_TMP"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.html"
        echo -e $url "\n"

        echo -e "After you verified that all the tables have been correctly produced, " \
            "they can be moved to the final repository using:"
        echo -e "    heds add2Web <taskFolder> \n"


        echo "WARNING: if some php pages are not opened corretly, it may be because apache server is not"
        echo " running on the machine hosting your HEDS_WWWAREA_TMP area."
        echo "In such a case, copy the whole HEDS_WWWAREA_TMP where "
        echo "php can be interpreted correctly: e.g. your personal webSite hosted at CERN "
        echo ""
        echo "To remind you, currently your HEDS_WWWAREA_TMP folder is: "
        echo $HEDS_WWWAREA_TMP

        ;;

    
    6)  ### copying stuff to castor
        tarName=$HEDS_LABEL.log.tgz
        cd $HEDS_LOCALSTORE/$dailySkim/logs/
        if (($?));then
            echo "ERROR: failed to go to " $HEDS_LOCALSTORE/$dailySkim/logs/
        else
            echo "copying " $tarName " file to CASTOR area.."
            rfcp $tarName $HEDS_CASTORFOLDER/skimV$HEDS_SKIMVERSION
            echo -e "done \n"
            cd -
        fi
        ;;

    5a)
        echo ".... preparing event display server"
        mkdir $HEDS_LABEL.shots
        port=3005
        $HEDSPATH/scripts/eventDisplayServer.sh $port &
        ;;
    5b)
        echo "getting snapshots"
        port=3005
        file=$HEDS_LOCALSTORE/merged_$HEDS_LABEL.root
        echo "$file" | nc -w 10 localhost $port
        ;;
    5c)
        echo "moving snapshots"
        mv $HEDS_LABEL.shots $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/
        cp $HEDSPATH/scripts/indexFileForDetailedSnapShots.php \
            $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.shots/index.php
        ;;


    final)
        #TO BE FIXED. this call could written in a  much more compact way
        echo "WWW pages are going to be moved from the temporary repository"
        echo -e "("$HEDS_WWWAREA_TMP")\n"
        echo "to the permanent one"
        echo -e "("$HEDS_WWWAREA")\n"

        if [ ! -d $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION ]; then
            mkdir $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION
        fi
        
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableBody.shtm.BAK
        fi
        if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.php ]; then
        cp $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/tableHeader.shtm.BAK
        fi
        
        cp $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.php \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/
        cp $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/*.shtm \
            $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/

        if [ -e $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
            if [ -e $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents ]; then
                echo "WARNING: replacing folder in the official AFS area of the HiggsDQM."

                echo "Do you know what you are doing? And is this what you really want to do? [y/n]"
                read -p "$prompt" answer
                case "$answer" in
                    [yY1] )
                        echo "proceeding with the update of the table.."
                        ;;
                    [nN0] )
                        exit 0
                        ;;
                    *) 
                        echo -e "Incorrect answer. Assuming you meant to type \"No\" "
                        exit
                        ;;
                esac            
                rm -r -i $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
            fi
            cp -r $HEDS_WWWAREA_TMP/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents \
                $HEDS_WWWAREA/logs.SkimV$HEDS_SKIMVERSION/$HEDS_LABEL.listSkimmedEvents
        fi


        url="https://cms-project-higgsdqm.web.cern.ch/cms-project-higgsdqm/HiggsEDS"
        url=$url"/logs.SkimV$HEDS_SKIMVERSION/FirstPage.php"
        
        echo "The new tables created by you are accessible through the following URL: "
        echo $url
        ;;

    #####################  after the crab part is done ###################
    99) echo "Doing bakcup of $HEDS_WWWAREA_TMP"
        date=`date +%k\.%M\.%S_%e\.%m\.%Y`
        fileName="HiggsIES.BAK."$date".tgz"
        echo "file name: " $fileName
        tar -czvf $fileName $HEDS_WWWAREA_TMP
        mv $fileName $HEDS_WWWAREA_TMP/..
        ;;


    *)
        echo "option not defined"
        ;;
esac
Revision:	1.11
Committed:	Wed May 12 17:19:26 2010 UTC (14 years, 11 months ago) by mangano
Content type:	application/x-sh
Branch:	MAIN
CVS Tags:	V00-01-02
Changes since 1.10:	+3 -2 lines
Log Message:	try to guess the username for crab jobs and DBS.