L1Menu/src/ReducedMenuSample.cpp

#include "l1menu/ReducedMenuSample.h"

#include <vector>
#include <stdexcept>
#include <fcntl.h>
#include <algorithm>
#include <iostream>
#include "l1menu/IReducedEvent.h"
#include "l1menu/MenuSample.h"
#include "l1menu/TriggerMenu.h"
#include "l1menu/ITrigger.h"
#include "l1menu/tools.h"
#include "protobuf/l1menu.pb.h"
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/io/gzip_stream.h>

namespace // unnamed namespace
{
        class ReducedEventImplementation : public l1menu::IReducedEvent
        {
        public:
                virtual ~ReducedEventImplementation() {}
                virtual float parameterValue( size_t parameterNumber ) const {  return pProtobufEvent->threshold(parameterNumber); }
                virtual float weight() const { if( pProtobufEvent->has_weight() ) return pProtobufEvent->weight(); else return 1; }
                void setWeight( float newWeight ) { pProtobufEvent->set_weight(newWeight); }
                l1menuprotobuf::Event* pProtobufEvent;
        };

        /** @brief Sentry that closes a Unix file descriptor when it goes out of scope.
         * @author Mark Grimes (mark.grimes@bristol.ac.uk)
         * @date 07/Jun/2013
         */
        class UnixFileSentry
        {
        public:
                UnixFileSentry( int fileDescriptor ) : fileDescriptor_(fileDescriptor) {}
                ~UnixFileSentry() { close(fileDescriptor_); }
        private:
                int fileDescriptor_;
        };
}

namespace l1menu
{
        /** @brief Private members for the ReducedMenuSample class
         *
         * @author Mark Grimes (mark.grimes@bristol.ac.uk)
         * @date 28/May/2013
         */
        class ReducedMenuSamplePrivateMembers
        {
        private:
                l1menu::TriggerMenu mutableTriggerMenu_;
        public:
                ReducedMenuSamplePrivateMembers( const l1menu::TriggerMenu& newTriggerMenu );
                ReducedMenuSamplePrivateMembers( const std::string& filename );
                void copyMenuToProtobufSample();
                ::ReducedEventImplementation event;
                const l1menu::TriggerMenu& triggerMenu;
                l1menuprotobuf::SampleHeader protobufSampleHeader;
                // Protobuf doesn't implement move semantics so I'll use pointers
                std::vector<std::unique_ptr<l1menuprotobuf::Run> > protobufRuns;
                const static int EVENTS_PER_RUN;
                const static char PROTOBUF_MESSAGE_DELIMETER;
                const static std::string FILE_FORMAT_MAGIC_NUMBER;
        };

        const int ReducedMenuSamplePrivateMembers::EVENTS_PER_RUN=20000;
        const char ReducedMenuSamplePrivateMembers::PROTOBUF_MESSAGE_DELIMETER='\n';
        const std::string ReducedMenuSamplePrivateMembers::FILE_FORMAT_MAGIC_NUMBER="l1menuReducedMenuSample";
}

l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const l1menu::TriggerMenu& newTriggerMenu )
        : mutableTriggerMenu_( newTriggerMenu ), triggerMenu( mutableTriggerMenu_ )
{
        GOOGLE_PROTOBUF_VERIFY_VERSION;

        // I need to copy the details of the trigger menu into the protobuf storage.
        // This means I'm holding a duplicate, but I need it to write the sample to a
        // protobuf file, so I might as well do it now.
        for( size_t triggerNumber=0; triggerNumber<triggerMenu.numberOfTriggers(); ++triggerNumber )
        {
                const l1menu::ITrigger& trigger=triggerMenu.getTrigger(triggerNumber);

                l1menuprotobuf::Trigger* pProtobufTrigger=protobufSampleHeader.add_trigger();
                pProtobufTrigger->set_name( trigger.name() );
                pProtobufTrigger->set_version( trigger.version() );

                // Record all of the parameters. It's not strictly necessary to record the values
                // of the parameters that are recorded for each event, but I might as well so that
                // the trigger menu is loaded exactly as it was saved.
                const auto parameterNames=trigger.parameterNames();
                for( const auto& parameterName : parameterNames )
                {
                        l1menuprotobuf::Trigger_TriggerParameter* pProtobufParameter=pProtobufTrigger->add_parameter();
                        pProtobufParameter->set_name(parameterName);
                        pProtobufParameter->set_value( trigger.parameter(parameterName) );
                }

                // Make a note of the names of the parameters that are recorded for each event. For this
                // I'm just recording the parameters that refer to the thresholds.
                const auto thresholdNames=l1menu::getThresholdNames(trigger);
                for( const auto& thresholdName : thresholdNames ) pProtobufTrigger->add_varying_parameter(thresholdName);

        } // end of loop over triggers

        // Always make sure there is at least one Run ready to be added to
        std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
        protobufRuns.push_back( std::move( pNewRun ) );

}

l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const std::string& filename )
        : triggerMenu(mutableTriggerMenu_)
{
        GOOGLE_PROTOBUF_VERIFY_VERSION;

        // Open the file with read ability
        int fileDescriptor = open( filename.c_str(), O_RDONLY );
        if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample initialise from file - couldn't open file" );
        ::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the input file
        google::protobuf::io::FileInputStream fileInput( fileDescriptor );
        google::protobuf::io::GzipInputStream gzipInput( &fileInput );
        google::protobuf::io::CodedInputStream codedInput( &gzipInput );

        // Disable warnings on this input stream (second parameter, -1). The
        // first parameter is the default. I'll change this if necessary in
        // the loop later.
        size_t totalBytesLimit=67108864;
        codedInput.SetTotalBytesLimit( totalBytesLimit, -1 );

        // First read the magic number at the start of the file and make sure it
        // matches what I expect. As a read buffer, I'll create a string the correct
        // size (filled with an arbitrary character) and read straight into that.
        std::string readMagicNumber;
        if( !codedInput.ReadString( &readMagicNumber, FILE_FORMAT_MAGIC_NUMBER.size() ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading magic number" );
        if( readMagicNumber!=FILE_FORMAT_MAGIC_NUMBER ) throw std::runtime_error( "ReducedMenuSample - tried to initialise with a file that is not the correct format" );

        google::protobuf::uint32 fileformatVersion;
        if( !codedInput.ReadVarint32( &fileformatVersion ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading file format version" );
        // So far I only have (and ever expect to have) one version of the file
        // format, imaginatively versioned "1". You never know though...
        if( fileformatVersion>1 ) std::cerr << "Warning: Attempting to read a ReducedMenuSample with version " << fileformatVersion << " with code that only knows up to version 1." << std::endl;

        google::protobuf::uint64 messageSize;

        // Read the size of the header message
        if( !codedInput.ReadVarint64( &messageSize ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading message size for header" );
        google::protobuf::io::CodedInputStream::Limit readLimit=codedInput.PushLimit(messageSize);
        if( !protobufSampleHeader.ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading header" );
        codedInput.PopLimit(readLimit);

        // Keep looping until there is nothing more to be read from the file.
        while( codedInput.ReadVarint64( &messageSize ) )
        {
                readLimit=codedInput.PushLimit(messageSize);

                // Make sure the CodedInputStream doesn't refuse to read the message because it's
                // read too much already. I'll also add an arbitrary 50 on to always make sure
                // I can read the next messageSize if there is one.
                if( gzipInput.ByteCount()+messageSize+50 > totalBytesLimit )
                {
                        totalBytesLimit+=messageSize*5; // Might as well set it a little higher than necessary while I'm at it.
                        codedInput.SetTotalBytesLimit( totalBytesLimit, -1 );
                }
                std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
                if( !pNewRun->ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading run" );
                protobufRuns.push_back( std::move( pNewRun ) );

                codedInput.PopLimit(readLimit);
        }


        // Always make sure there is at least one Run ready to be added to. Later
        // code assumes there is already a run there.
        if( protobufRuns.empty() )
        {
                std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
                protobufRuns.push_back( std::move( pNewRun ) );
        }

        // I have all of the information in the protobuf members, but I also need the trigger information
        // in the form of l1menu::TriggerMenu. Copy out the required information.
        for( int triggerNumber=0; triggerNumber<protobufSampleHeader.trigger_size(); ++triggerNumber )
        {
                const l1menuprotobuf::Trigger& inputTrigger=protobufSampleHeader.trigger(triggerNumber);

                mutableTriggerMenu_.addTrigger( inputTrigger.name(), inputTrigger.version() );
                // Get a reference to the trigger I just created
                l1menu::ITrigger& trigger=mutableTriggerMenu_.getTrigger(triggerNumber);

                // Run through all of the parameters and set them to what they were
                // when the sample was made.
                for( int parameterNumber=0; parameterNumber<inputTrigger.parameter_size(); ++parameterNumber )
                {
                        const auto& inputParameter=inputTrigger.parameter(parameterNumber);
                        trigger.parameter(inputParameter.name())=inputParameter.value();
                }

                // I should probably check the threshold names exist. I'll do it another time.
        }

}

l1menu::ReducedMenuSample::ReducedMenuSample( const l1menu::MenuSample& originalSample, const l1menu::TriggerMenu& triggerMenu )
        : pImple_( new l1menu::ReducedMenuSamplePrivateMembers( triggerMenu ) )
{
        addSample( originalSample );
}

l1menu::ReducedMenuSample::ReducedMenuSample( const l1menu::TriggerMenu& triggerMenu )
        : pImple_( new l1menu::ReducedMenuSamplePrivateMembers( triggerMenu ) )
{
        // No operation besides the initialiser list
}

l1menu::ReducedMenuSample::~ReducedMenuSample()
{
        // No operation. Just need one defined otherwise the default one messes up
        // the unique_ptr deletion because ReducedMenuSamplePrivateMembers isn't
        // defined elsewhere.
}

void l1menu::ReducedMenuSample::addSample( const l1menu::MenuSample& originalSample )
{
        l1menuprotobuf::Run* pCurrentRun=pImple_->protobufRuns.back().get();

        for( size_t eventNumber=0; eventNumber<originalSample.numberOfEvents(); ++eventNumber )
        {
                // Split the events up into groups in arbitrary numbers. This is to get around
                // a protobuf aversion to long messages.
                if( pCurrentRun->event_size() >= pImple_->EVENTS_PER_RUN )
                {
                        // Gone over the arbitrary limit, so create a new protobuf Run and start
                        // using that instead.
                        std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
                        pImple_->protobufRuns.push_back( std::move( pNewRun ) );
                        pCurrentRun=pImple_->protobufRuns.back().get();
                }

                const l1menu::IEvent& event=originalSample.getEvent( eventNumber );
                l1menuprotobuf::Event* pProtobufEvent=pCurrentRun->add_event();

                // Loop over all of the triggers
                for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
                {
                        std::unique_ptr<l1menu::ITrigger> pTrigger=pImple_->triggerMenu.getTriggerCopy(triggerNumber);
                        std::vector<std::string> thresholdNames=getThresholdNames(*pTrigger);

                        try
                        {
                                setTriggerThresholdsAsTightAsPossible( event, *pTrigger, 0.001 );
                                // Set all of the parameters to match the thresholds in the trigger
                                for( const auto& thresholdName : thresholdNames )
                                {
                                        pProtobufEvent->add_threshold( pTrigger->parameter(thresholdName) );
                                }
                        }
                        catch( std::exception& error )
                        {
                                // setTriggerThresholdsAsTightAsPossible() couldn't find thresholds so record
                                // -1 for everything.
                                // Range based for loop gives me a warning because I don't use the thresholdName.
                                for( size_t index=0; index<thresholdNames.size(); ++index ) pProtobufEvent->add_threshold(-1);
                        } // end of try block that sets the trigger thresholds

                } // end of loop over triggers

        } // end of loop over events
}

l1menu::ReducedMenuSample::ReducedMenuSample( const std::string& filename )
        : pImple_( new l1menu::ReducedMenuSamplePrivateMembers( filename ) )
{
        // No operation except the initialiser list
}

void l1menu::ReducedMenuSample::saveToFile( const std::string& filename ) const
{
        // Open the file. Parameters are filename, write ability and create, rw-r--r-- permissions.
        int fileDescriptor = open( filename.c_str(), O_WRONLY | O_CREAT, 0644 );
        if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample save to file - couldn't open file" );
        ::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the output file

        // Setup the protobuf file handlers
        google::protobuf::io::FileOutputStream fileOutput( fileDescriptor );
        google::protobuf::io::GzipOutputStream gzipOutput( &fileOutput );
        google::protobuf::io::CodedOutputStream codedOutput( &gzipOutput );

        // Write a magic number at the start of all files
        codedOutput.WriteString( pImple_->FILE_FORMAT_MAGIC_NUMBER );
        // Write an integer that specifies what version of the file format I'm using. I
        // have no intention of changing the format but I might as well keep the option
        // open.
        codedOutput.WriteVarint32( 1 );

        // Write the size of the header message into the file...
        codedOutput.WriteVarint64( pImple_->protobufSampleHeader.ByteSize() );
        // ...and then write the header
        pImple_->protobufSampleHeader.SerializeToCodedStream( &codedOutput );

        // Now go through each of the runs and do the same for those
        for( const auto& pRun : pImple_->protobufRuns )
        {
                codedOutput.WriteVarint64( pRun->ByteSize() );
                pRun->SerializeToCodedStream( &codedOutput );
        }

}

size_t l1menu::ReducedMenuSample::numberOfEvents() const
{
        size_t numberOfEvents=0;
        for( const auto& pRun : pImple_->protobufRuns ) numberOfEvents+=pRun->event_size();
        return numberOfEvents;
}

const l1menu::IReducedEvent& l1menu::ReducedMenuSample::getEvent( size_t eventNumber ) const
{
        for( const auto& pRun : pImple_->protobufRuns )
        {
                if( eventNumber<static_cast<size_t>(pRun->event_size()) )
                {
                        pImple_->event.pProtobufEvent=pRun->mutable_event(eventNumber);
                        return pImple_->event;
                }
                // Event must be in a later run, so reduce the number by how many events
                // were in this run and look again.
                eventNumber-=pRun->event_size();
        }

        // Should always find the event before getting to this point, so throw an
        // exception if this happens.
        throw std::runtime_error( "ReducedMenuSample::getEvent(eventNumber) was asked for an invalid eventNumber" );
}

const l1menu::TriggerMenu& l1menu::ReducedMenuSample::getTriggerMenu() const
{
        return pImple_->triggerMenu;
}

bool l1menu::ReducedMenuSample::containsTrigger( const l1menu::ITrigger& trigger, bool allowOlderVersion ) const
{
        // Loop over all of the triggers in the menu, and see if there is one
        // where the name and version match.
        for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
        {
                const l1menu::ITrigger& triggerInMenu=pImple_->triggerMenu.getTrigger(triggerNumber);
                if( triggerInMenu.name()!=trigger.name() ) continue;
                if( allowOlderVersion )
                {
                        if( triggerInMenu.version()>trigger.version() ) continue;
                }
                else
                {
                        if( triggerInMenu.version()!=trigger.version() ) continue;
                }

                // If control got this far then there is a trigger with the required name
                // and sufficient version. I now need to check all of the non threshold parameters
                // to make sure they match, i.e. make sure the ReducedSample was made with the same
                // eta cuts or whatever.
                // I don't care if the thresholds don't match because that's what's stored in the
                // ReducedMenuSample.
                std::vector<std::string> parameterNames=getNonThresholdParameterNames( trigger );
                bool allParametersMatch=true;
                for( const auto& parameterName : parameterNames )
                {
                        if( trigger.parameter(parameterName)!=triggerInMenu.parameter(parameterName) ) allParametersMatch=false;
                }

                if( allParametersMatch ) return true;
        } // end of loop over triggers

        // If control got this far then no trigger was found that matched
        return false;
}

const std::map<std::string,size_t> l1menu::ReducedMenuSample::getTriggerParameterIdentifiers( const l1menu::ITrigger& trigger, bool allowOlderVersion ) const
{
        std::map<std::string,size_t> returnValue;

        // Need to find out how many parameters there are for each event. Basically the sum
        // of the number of thresholds for all triggers.
        size_t parameterNumber=0;
        bool triggerWasFound=true;
        for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
        {
                const l1menu::ITrigger& triggerInMenu=pImple_->triggerMenu.getTrigger(triggerNumber);

                triggerWasFound=true; // Set to true, then back to false if any of the tests fail
                // See if this trigger in the menu is the same as the one passed as a parameter
                if( triggerInMenu.name()!=trigger.name() ) triggerWasFound=false;
                if( allowOlderVersion )
                {
                        if( triggerInMenu.version()>trigger.version() ) triggerWasFound=false;
                }
                else
                {
                        if( triggerInMenu.version()!=trigger.version() ) triggerWasFound=false;
                }

                // If control got this far then there is a trigger with the required name
                // and sufficient version. I now need to check all of the non threshold parameters
                // to make sure they match, i.e. make sure the ReducedSample was made with the same
                // eta cuts or whatever.
                // I don't care if the thresholds don't match because that's what's stored in the
                // ReducedMenuSample.
                if( triggerWasFound ) // Trigger can still fail, but no point doing this check if it already has
                {
                        std::vector<std::string> parameterNames=getNonThresholdParameterNames( trigger );
                        for( const auto& parameterName : parameterNames )
                        {
                                if( trigger.parameter(parameterName)!=triggerInMenu.parameter(parameterName) ) triggerWasFound=false;
                        }
                }

                std::vector<std::string> thresholdNames=l1menu::getThresholdNames(triggerInMenu);
                if( triggerWasFound )
                {
                        for( const auto& thresholdName : thresholdNames )
                        {
                                returnValue[thresholdName]=parameterNumber;
                                ++parameterNumber;
                        }
                        break;
                }
                else parameterNumber+=thresholdNames.size();
        }

        // There could conceivably be a trigger that was found but has no thresholds
        // (I guess - it would be a pretty pointless trigger though). To indicate the
        // difference between that and a trigger that wasn't found I'll respectively
        // return the empty vector or throw an exception.
        if( !triggerWasFound ) throw std::runtime_error( "l1menu::ReducedMenuSample::getTriggerParameterIdentifiers() called for a trigger that was not used to create the sample" );

        return returnValue;
}
Revision:	1.9
Committed:	Sat Jun 8 09:31:25 2013 UTC (11 years, 10 months ago) by grimes
Branch:	MAIN
Changes since 1.8:	+15 -1 lines
Log Message:	Changes to load very large files, and some fixes about histogram binning.
#	User	Rev	Content
1	grimes	1.1	#include "l1menu/ReducedMenuSample.h"
2
3			#include <vector>
4	grimes	1.2	#include <stdexcept>
5	grimes	1.8	#include <fcntl.h>
6	grimes	1.7	#include <algorithm>
7	grimes	1.8	#include <iostream>
8	grimes	1.1	#include "l1menu/IReducedEvent.h"
9			#include "l1menu/MenuSample.h"
10			#include "l1menu/TriggerMenu.h"
11	grimes	1.2	#include "l1menu/ITrigger.h"
12	grimes	1.1	#include "l1menu/tools.h"
13	grimes	1.2	#include "protobuf/l1menu.pb.h"
14	grimes	1.8	#include <google/protobuf/io/zero_copy_stream_impl.h>
15			#include <google/protobuf/io/gzip_stream.h>
16	grimes	1.1
17			namespace // unnamed namespace
18			{
19			class ReducedEventImplementation : public l1menu::IReducedEvent
20			{
21			public:
22			virtual ~ReducedEventImplementation() {}
23	grimes	1.7	virtual float parameterValue( size_t parameterNumber ) const { return pProtobufEvent->threshold(parameterNumber); }
24			virtual float weight() const { if( pProtobufEvent->has_weight() ) return pProtobufEvent->weight(); else return 1; }
25			void setWeight( float newWeight ) { pProtobufEvent->set_weight(newWeight); }
26			l1menuprotobuf::Event* pProtobufEvent;
27	grimes	1.1	};
28	grimes	1.8
29			/** @brief Sentry that closes a Unix file descriptor when it goes out of scope.
30			* @author Mark Grimes (mark.grimes@bristol.ac.uk)
31			* @date 07/Jun/2013
32			*/
33			class UnixFileSentry
34			{
35			public:
36			UnixFileSentry( int fileDescriptor ) : fileDescriptor_(fileDescriptor) {}
37			~UnixFileSentry() { close(fileDescriptor_); }
38			private:
39			int fileDescriptor_;
40			};
41	grimes	1.1	}
42
43			namespace l1menu
44			{
45			/** @brief Private members for the ReducedMenuSample class
46			*
47			* @author Mark Grimes (mark.grimes@bristol.ac.uk)
48			* @date 28/May/2013
49			*/
50			class ReducedMenuSamplePrivateMembers
51			{
52	grimes	1.7	private:
53			l1menu::TriggerMenu mutableTriggerMenu_;
54	grimes	1.1	public:
55	grimes	1.7	ReducedMenuSamplePrivateMembers( const l1menu::TriggerMenu& newTriggerMenu );
56			ReducedMenuSamplePrivateMembers( const std::string& filename );
57			void copyMenuToProtobufSample();
58	grimes	1.1	::ReducedEventImplementation event;
59	grimes	1.7	const l1menu::TriggerMenu& triggerMenu;
60	grimes	1.8	l1menuprotobuf::SampleHeader protobufSampleHeader;
61			// Protobuf doesn't implement move semantics so I'll use pointers
62			std::vector<std::unique_ptr<l1menuprotobuf::Run> > protobufRuns;
63			const static int EVENTS_PER_RUN;
64			const static char PROTOBUF_MESSAGE_DELIMETER;
65			const static std::string FILE_FORMAT_MAGIC_NUMBER;
66	grimes	1.1	};
67	grimes	1.8
68			const int ReducedMenuSamplePrivateMembers::EVENTS_PER_RUN=20000;
69			const char ReducedMenuSamplePrivateMembers::PROTOBUF_MESSAGE_DELIMETER='\n';
70			const std::string ReducedMenuSamplePrivateMembers::FILE_FORMAT_MAGIC_NUMBER="l1menuReducedMenuSample";
71	grimes	1.1	}
72
73	grimes	1.7	l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const l1menu::TriggerMenu& newTriggerMenu )
74			: mutableTriggerMenu_( newTriggerMenu ), triggerMenu( mutableTriggerMenu_ )
75	grimes	1.1	{
76	grimes	1.7	GOOGLE_PROTOBUF_VERIFY_VERSION;
77
78			// I need to copy the details of the trigger menu into the protobuf storage.
79	grimes	1.8	// This means I'm holding a duplicate, but I need it to write the sample to a
80			// protobuf file, so I might as well do it now.
81	grimes	1.1	for( size_t triggerNumber=0; triggerNumber<triggerMenu.numberOfTriggers(); ++triggerNumber )
82			{
83			const l1menu::ITrigger& trigger=triggerMenu.getTrigger(triggerNumber);
84	grimes	1.7
85	grimes	1.8	l1menuprotobuf::Trigger* pProtobufTrigger=protobufSampleHeader.add_trigger();
86	grimes	1.7	pProtobufTrigger->set_name( trigger.name() );
87			pProtobufTrigger->set_version( trigger.version() );
88
89			// Record all of the parameters. It's not strictly necessary to record the values
90			// of the parameters that are recorded for each event, but I might as well so that
91			// the trigger menu is loaded exactly as it was saved.
92			const auto parameterNames=trigger.parameterNames();
93			for( const auto& parameterName : parameterNames )
94			{
95			l1menuprotobuf::Trigger_TriggerParameter* pProtobufParameter=pProtobufTrigger->add_parameter();
96			pProtobufParameter->set_name(parameterName);
97			pProtobufParameter->set_value( trigger.parameter(parameterName) );
98			}
99
100			// Make a note of the names of the parameters that are recorded for each event. For this
101			// I'm just recording the parameters that refer to the thresholds.
102			const auto thresholdNames=l1menu::getThresholdNames(trigger);
103	grimes	1.8	for( const auto& thresholdName : thresholdNames ) pProtobufTrigger->add_varying_parameter(thresholdName);
104	grimes	1.7
105			} // end of loop over triggers
106
107	grimes	1.8	// Always make sure there is at least one Run ready to be added to
108			std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
109			protobufRuns.push_back( std::move( pNewRun ) );
110
111	grimes	1.7	}
112
113			l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const std::string& filename )
114			: triggerMenu(mutableTriggerMenu_)
115			{
116			GOOGLE_PROTOBUF_VERIFY_VERSION;
117
118	grimes	1.8	// Open the file with read ability
119			int fileDescriptor = open( filename.c_str(), O_RDONLY );
120			if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample initialise from file - couldn't open file" );
121			::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the input file
122			google::protobuf::io::FileInputStream fileInput( fileDescriptor );
123			google::protobuf::io::GzipInputStream gzipInput( &fileInput );
124			google::protobuf::io::CodedInputStream codedInput( &gzipInput );
125
126	grimes	1.9	// Disable warnings on this input stream (second parameter, -1). The
127			// first parameter is the default. I'll change this if necessary in
128			// the loop later.
129			size_t totalBytesLimit=67108864;
130			codedInput.SetTotalBytesLimit( totalBytesLimit, -1 );
131
132	grimes	1.8	// First read the magic number at the start of the file and make sure it
133			// matches what I expect. As a read buffer, I'll create a string the correct
134			// size (filled with an arbitrary character) and read straight into that.
135			std::string readMagicNumber;
136			if( !codedInput.ReadString( &readMagicNumber, FILE_FORMAT_MAGIC_NUMBER.size() ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading magic number" );
137			if( readMagicNumber!=FILE_FORMAT_MAGIC_NUMBER ) throw std::runtime_error( "ReducedMenuSample - tried to initialise with a file that is not the correct format" );
138
139			google::protobuf::uint32 fileformatVersion;
140			if( !codedInput.ReadVarint32( &fileformatVersion ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading file format version" );
141			// So far I only have (and ever expect to have) one version of the file
142			// format, imaginatively versioned "1". You never know though...
143			if( fileformatVersion>1 ) std::cerr << "Warning: Attempting to read a ReducedMenuSample with version " << fileformatVersion << " with code that only knows up to version 1." << std::endl;
144
145			google::protobuf::uint64 messageSize;
146
147			// Read the size of the header message
148			if( !codedInput.ReadVarint64( &messageSize ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading message size for header" );
149			google::protobuf::io::CodedInputStream::Limit readLimit=codedInput.PushLimit(messageSize);
150			if( !protobufSampleHeader.ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading header" );
151			codedInput.PopLimit(readLimit);
152
153			// Keep looping until there is nothing more to be read from the file.
154			while( codedInput.ReadVarint64( &messageSize ) )
155			{
156			readLimit=codedInput.PushLimit(messageSize);
157
158	grimes	1.9	// Make sure the CodedInputStream doesn't refuse to read the message because it's
159			// read too much already. I'll also add an arbitrary 50 on to always make sure
160			// I can read the next messageSize if there is one.
161			if( gzipInput.ByteCount()+messageSize+50 > totalBytesLimit )
162			{
163			totalBytesLimit+=messageSize*5; // Might as well set it a little higher than necessary while I'm at it.
164			codedInput.SetTotalBytesLimit( totalBytesLimit, -1 );
165			}
166	grimes	1.8	std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
167			if( !pNewRun->ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading run" );
168			protobufRuns.push_back( std::move( pNewRun ) );
169
170			codedInput.PopLimit(readLimit);
171	grimes	1.1	}
172
173	grimes	1.7
174	grimes	1.8	// Always make sure there is at least one Run ready to be added to. Later
175			// code assumes there is already a run there.
176			if( protobufRuns.empty() )
177			{
178			std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
179			protobufRuns.push_back( std::move( pNewRun ) );
180			}
181
182			// I have all of the information in the protobuf members, but I also need the trigger information
183	grimes	1.7	// in the form of l1menu::TriggerMenu. Copy out the required information.
184	grimes	1.8	for( int triggerNumber=0; triggerNumber<protobufSampleHeader.trigger_size(); ++triggerNumber )
185	grimes	1.2	{
186	grimes	1.8	const l1menuprotobuf::Trigger& inputTrigger=protobufSampleHeader.trigger(triggerNumber);
187	grimes	1.7
188			mutableTriggerMenu_.addTrigger( inputTrigger.name(), inputTrigger.version() );
189			// Get a reference to the trigger I just created
190			l1menu::ITrigger& trigger=mutableTriggerMenu_.getTrigger(triggerNumber);
191	grimes	1.2
192	grimes	1.7	// Run through all of the parameters and set them to what they were
193			// when the sample was made.
194			for( int parameterNumber=0; parameterNumber<inputTrigger.parameter_size(); ++parameterNumber )
195	grimes	1.2	{
196	grimes	1.7	const auto& inputParameter=inputTrigger.parameter(parameterNumber);
197			trigger.parameter(inputParameter.name())=inputParameter.value();
198			}
199
200			// I should probably check the threshold names exist. I'll do it another time.
201			}
202	grimes	1.2
203	grimes	1.7	}
204	grimes	1.2
205	grimes	1.7	l1menu::ReducedMenuSample::ReducedMenuSample( const l1menu::MenuSample& originalSample, const l1menu::TriggerMenu& triggerMenu )
206			: pImple_( new l1menu::ReducedMenuSamplePrivateMembers( triggerMenu ) )
207			{
208			addSample( originalSample );
209	grimes	1.2	}
210
211	grimes	1.5	l1menu::ReducedMenuSample::ReducedMenuSample( const l1menu::TriggerMenu& triggerMenu )
212	grimes	1.7	: pImple_( new l1menu::ReducedMenuSamplePrivateMembers( triggerMenu ) )
213	grimes	1.5	{
214	grimes	1.7	// No operation besides the initialiser list
215	grimes	1.5	}
216
217	grimes	1.2	l1menu::ReducedMenuSample::~ReducedMenuSample()
218			{
219			// No operation. Just need one defined otherwise the default one messes up
220			// the unique_ptr deletion because ReducedMenuSamplePrivateMembers isn't
221			// defined elsewhere.
222			}
223
224	grimes	1.5	void l1menu::ReducedMenuSample::addSample( const l1menu::MenuSample& originalSample )
225			{
226	grimes	1.8	l1menuprotobuf::Run* pCurrentRun=pImple_->protobufRuns.back().get();
227
228	grimes	1.5	for( size_t eventNumber=0; eventNumber<originalSample.numberOfEvents(); ++eventNumber )
229			{
230	grimes	1.8	// Split the events up into groups in arbitrary numbers. This is to get around
231			// a protobuf aversion to long messages.
232	grimes	1.9	if( pCurrentRun->event_size() >= pImple_->EVENTS_PER_RUN )
233	grimes	1.8	{
234			// Gone over the arbitrary limit, so create a new protobuf Run and start
235			// using that instead.
236			std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run );
237			pImple_->protobufRuns.push_back( std::move( pNewRun ) );
238			pCurrentRun=pImple_->protobufRuns.back().get();
239			}
240
241	grimes	1.5	const l1menu::IEvent& event=originalSample.getEvent( eventNumber );
242	grimes	1.8	l1menuprotobuf::Event* pProtobufEvent=pCurrentRun->add_event();
243	grimes	1.5
244			// Loop over all of the triggers
245			for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
246			{
247			std::unique_ptr<l1menu::ITrigger> pTrigger=pImple_->triggerMenu.getTriggerCopy(triggerNumber);
248			std::vector<std::string> thresholdNames=getThresholdNames(*pTrigger);
249
250			try
251			{
252			setTriggerThresholdsAsTightAsPossible( event, *pTrigger, 0.001 );
253			// Set all of the parameters to match the thresholds in the trigger
254			for( const auto& thresholdName : thresholdNames )
255			{
256	grimes	1.7	pProtobufEvent->add_threshold( pTrigger->parameter(thresholdName) );
257	grimes	1.5	}
258			}
259			catch( std::exception& error )
260			{
261			// setTriggerThresholdsAsTightAsPossible() couldn't find thresholds so record
262			// -1 for everything.
263	grimes	1.7	// Range based for loop gives me a warning because I don't use the thresholdName.
264			for( size_t index=0; index<thresholdNames.size(); ++index ) pProtobufEvent->add_threshold(-1);
265	grimes	1.5	} // end of try block that sets the trigger thresholds
266
267			} // end of loop over triggers
268	grimes	1.7
269	grimes	1.5	} // end of loop over events
270			}
271
272	grimes	1.2	l1menu::ReducedMenuSample::ReducedMenuSample( const std::string& filename )
273	grimes	1.7	: pImple_( new l1menu::ReducedMenuSamplePrivateMembers( filename ) )
274	grimes	1.2	{
275	grimes	1.7	// No operation except the initialiser list
276	grimes	1.2	}
277
278			void l1menu::ReducedMenuSample::saveToFile( const std::string& filename ) const
279			{
280	grimes	1.8	// Open the file. Parameters are filename, write ability and create, rw-r--r-- permissions.
281			int fileDescriptor = open( filename.c_str(), O_WRONLY \| O_CREAT, 0644 );
282			if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample save to file - couldn't open file" );
283			::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the output file
284
285			// Setup the protobuf file handlers
286			google::protobuf::io::FileOutputStream fileOutput( fileDescriptor );
287			google::protobuf::io::GzipOutputStream gzipOutput( &fileOutput );
288			google::protobuf::io::CodedOutputStream codedOutput( &gzipOutput );
289
290			// Write a magic number at the start of all files
291			codedOutput.WriteString( pImple_->FILE_FORMAT_MAGIC_NUMBER );
292			// Write an integer that specifies what version of the file format I'm using. I
293			// have no intention of changing the format but I might as well keep the option
294			// open.
295			codedOutput.WriteVarint32( 1 );
296
297			// Write the size of the header message into the file...
298			codedOutput.WriteVarint64( pImple_->protobufSampleHeader.ByteSize() );
299			// ...and then write the header
300			pImple_->protobufSampleHeader.SerializeToCodedStream( &codedOutput );
301
302			// Now go through each of the runs and do the same for those
303			for( const auto& pRun : pImple_->protobufRuns )
304			{
305			codedOutput.WriteVarint64( pRun->ByteSize() );
306			pRun->SerializeToCodedStream( &codedOutput );
307			}
308
309	grimes	1.1	}
310
311			size_t l1menu::ReducedMenuSample::numberOfEvents() const
312			{
313	grimes	1.8	size_t numberOfEvents=0;
314			for( const auto& pRun : pImple_->protobufRuns ) numberOfEvents+=pRun->event_size();
315			return numberOfEvents;
316	grimes	1.1	}
317
318			const l1menu::IReducedEvent& l1menu::ReducedMenuSample::getEvent( size_t eventNumber ) const
319			{
320	grimes	1.8	for( const auto& pRun : pImple_->protobufRuns )
321			{
322			if( eventNumber<static_cast<size_t>(pRun->event_size()) )
323			{
324			pImple_->event.pProtobufEvent=pRun->mutable_event(eventNumber);
325			return pImple_->event;
326			}
327			// Event must be in a later run, so reduce the number by how many events
328			// were in this run and look again.
329			eventNumber-=pRun->event_size();
330			}
331
332			// Should always find the event before getting to this point, so throw an
333			// exception if this happens.
334			throw std::runtime_error( "ReducedMenuSample::getEvent(eventNumber) was asked for an invalid eventNumber" );
335	grimes	1.1	}
336	grimes	1.2
337			const l1menu::TriggerMenu& l1menu::ReducedMenuSample::getTriggerMenu() const
338			{
339			return pImple_->triggerMenu;
340			}
341
342			bool l1menu::ReducedMenuSample::containsTrigger( const l1menu::ITrigger& trigger, bool allowOlderVersion ) const
343			{
344			// Loop over all of the triggers in the menu, and see if there is one
345			// where the name and version match.
346			for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
347			{
348			const l1menu::ITrigger& triggerInMenu=pImple_->triggerMenu.getTrigger(triggerNumber);
349			if( triggerInMenu.name()!=trigger.name() ) continue;
350			if( allowOlderVersion )
351			{
352			if( triggerInMenu.version()>trigger.version() ) continue;
353			}
354			else
355			{
356			if( triggerInMenu.version()!=trigger.version() ) continue;
357			}
358
359			// If control got this far then there is a trigger with the required name
360			// and sufficient version. I now need to check all of the non threshold parameters
361			// to make sure they match, i.e. make sure the ReducedSample was made with the same
362			// eta cuts or whatever.
363			// I don't care if the thresholds don't match because that's what's stored in the
364			// ReducedMenuSample.
365			std::vector<std::string> parameterNames=getNonThresholdParameterNames( trigger );
366			bool allParametersMatch=true;
367			for( const auto& parameterName : parameterNames )
368			{
369			if( trigger.parameter(parameterName)!=triggerInMenu.parameter(parameterName) ) allParametersMatch=false;
370			}
371
372			if( allParametersMatch ) return true;
373			} // end of loop over triggers
374
375			// If control got this far then no trigger was found that matched
376			return false;
377			}
378
379			const std::map<std::string,size_t> l1menu::ReducedMenuSample::getTriggerParameterIdentifiers( const l1menu::ITrigger& trigger, bool allowOlderVersion ) const
380			{
381			std::map<std::string,size_t> returnValue;
382
383			// Need to find out how many parameters there are for each event. Basically the sum
384			// of the number of thresholds for all triggers.
385			size_t parameterNumber=0;
386			bool triggerWasFound=true;
387			for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber )
388			{
389			const l1menu::ITrigger& triggerInMenu=pImple_->triggerMenu.getTrigger(triggerNumber);
390
391			triggerWasFound=true; // Set to true, then back to false if any of the tests fail
392			// See if this trigger in the menu is the same as the one passed as a parameter
393			if( triggerInMenu.name()!=trigger.name() ) triggerWasFound=false;
394			if( allowOlderVersion )
395			{
396			if( triggerInMenu.version()>trigger.version() ) triggerWasFound=false;
397			}
398			else
399			{
400			if( triggerInMenu.version()!=trigger.version() ) triggerWasFound=false;
401			}
402
403			// If control got this far then there is a trigger with the required name
404			// and sufficient version. I now need to check all of the non threshold parameters
405			// to make sure they match, i.e. make sure the ReducedSample was made with the same
406			// eta cuts or whatever.
407			// I don't care if the thresholds don't match because that's what's stored in the
408			// ReducedMenuSample.
409			if( triggerWasFound ) // Trigger can still fail, but no point doing this check if it already has
410			{
411			std::vector<std::string> parameterNames=getNonThresholdParameterNames( trigger );
412			for( const auto& parameterName : parameterNames )
413			{
414			if( trigger.parameter(parameterName)!=triggerInMenu.parameter(parameterName) ) triggerWasFound=false;
415			}
416			}
417
418	grimes	1.4	std::vector<std::string> thresholdNames=l1menu::getThresholdNames(triggerInMenu);
419	grimes	1.2	if( triggerWasFound )
420			{
421			for( const auto& thresholdName : thresholdNames )
422			{
423			returnValue[thresholdName]=parameterNumber;
424			++parameterNumber;
425			}
426			break;
427			}
428			else parameterNumber+=thresholdNames.size();
429			}
430
431			// There could conceivably be a trigger that was found but has no thresholds
432			// (I guess - it would be a pretty pointless trigger though). To indicate the
433			// difference between that and a trigger that wasn't found I'll respectively
434			// return the empty vector or throw an exception.
435			if( !triggerWasFound ) throw std::runtime_error( "l1menu::ReducedMenuSample::getTriggerParameterIdentifiers() called for a trigger that was not used to create the sample" );
436
437			return returnValue;
438			}