| 2 |
|
|
| 3 |
|
#include <vector> |
| 4 |
|
#include <stdexcept> |
| 5 |
< |
#include <fstream> |
| 5 |
> |
#include <fcntl.h> |
| 6 |
|
#include <algorithm> |
| 7 |
+ |
#include <iostream> |
| 8 |
|
#include "l1menu/IReducedEvent.h" |
| 9 |
|
#include "l1menu/MenuSample.h" |
| 10 |
|
#include "l1menu/TriggerMenu.h" |
| 11 |
|
#include "l1menu/ITrigger.h" |
| 12 |
|
#include "l1menu/tools.h" |
| 13 |
|
#include "protobuf/l1menu.pb.h" |
| 14 |
+ |
#include <google/protobuf/io/zero_copy_stream_impl.h> |
| 15 |
+ |
#include <google/protobuf/io/gzip_stream.h> |
| 16 |
|
|
| 17 |
|
namespace // unnamed namespace |
| 18 |
|
{ |
| 25 |
|
void setWeight( float newWeight ) { pProtobufEvent->set_weight(newWeight); } |
| 26 |
|
l1menuprotobuf::Event* pProtobufEvent; |
| 27 |
|
}; |
| 28 |
+ |
|
| 29 |
+ |
/** @brief Sentry that closes a Unix file descriptor when it goes out of scope. |
| 30 |
+ |
* @author Mark Grimes (mark.grimes@bristol.ac.uk) |
| 31 |
+ |
* @date 07/Jun/2013 |
| 32 |
+ |
*/ |
| 33 |
+ |
class UnixFileSentry |
| 34 |
+ |
{ |
| 35 |
+ |
public: |
| 36 |
+ |
UnixFileSentry( int fileDescriptor ) : fileDescriptor_(fileDescriptor) {} |
| 37 |
+ |
~UnixFileSentry() { close(fileDescriptor_); } |
| 38 |
+ |
private: |
| 39 |
+ |
int fileDescriptor_; |
| 40 |
+ |
}; |
| 41 |
|
} |
| 42 |
|
|
| 43 |
|
namespace l1menu |
| 57 |
|
void copyMenuToProtobufSample(); |
| 58 |
|
::ReducedEventImplementation event; |
| 59 |
|
const l1menu::TriggerMenu& triggerMenu; |
| 60 |
< |
l1menuprotobuf::Sample protobufSample; |
| 60 |
> |
l1menuprotobuf::SampleHeader protobufSampleHeader; |
| 61 |
> |
// Protobuf doesn't implement move semantics so I'll use pointers |
| 62 |
> |
std::vector<std::unique_ptr<l1menuprotobuf::Run> > protobufRuns; |
| 63 |
> |
const static int EVENTS_PER_RUN; |
| 64 |
> |
const static char PROTOBUF_MESSAGE_DELIMETER; |
| 65 |
> |
const static std::string FILE_FORMAT_MAGIC_NUMBER; |
| 66 |
|
}; |
| 67 |
+ |
|
| 68 |
+ |
const int ReducedMenuSamplePrivateMembers::EVENTS_PER_RUN=20000; |
| 69 |
+ |
const char ReducedMenuSamplePrivateMembers::PROTOBUF_MESSAGE_DELIMETER='\n'; |
| 70 |
+ |
const std::string ReducedMenuSamplePrivateMembers::FILE_FORMAT_MAGIC_NUMBER="l1menuReducedMenuSample"; |
| 71 |
|
} |
| 72 |
|
|
| 73 |
|
l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const l1menu::TriggerMenu& newTriggerMenu ) |
| 76 |
|
GOOGLE_PROTOBUF_VERIFY_VERSION; |
| 77 |
|
|
| 78 |
|
// I need to copy the details of the trigger menu into the protobuf storage. |
| 79 |
< |
// This means I'm holding a duplicate, but I need it to write the Sample message |
| 80 |
< |
// so I might as well do it now. |
| 79 |
> |
// This means I'm holding a duplicate, but I need it to write the sample to a |
| 80 |
> |
// protobuf file, so I might as well do it now. |
| 81 |
|
for( size_t triggerNumber=0; triggerNumber<triggerMenu.numberOfTriggers(); ++triggerNumber ) |
| 82 |
|
{ |
| 83 |
|
const l1menu::ITrigger& trigger=triggerMenu.getTrigger(triggerNumber); |
| 84 |
|
|
| 85 |
< |
l1menuprotobuf::Trigger* pProtobufTrigger=protobufSample.add_trigger(); |
| 85 |
> |
l1menuprotobuf::Trigger* pProtobufTrigger=protobufSampleHeader.add_trigger(); |
| 86 |
|
pProtobufTrigger->set_name( trigger.name() ); |
| 87 |
|
pProtobufTrigger->set_version( trigger.version() ); |
| 88 |
|
|
| 100 |
|
// Make a note of the names of the parameters that are recorded for each event. For this |
| 101 |
|
// I'm just recording the parameters that refer to the thresholds. |
| 102 |
|
const auto thresholdNames=l1menu::getThresholdNames(trigger); |
| 103 |
< |
for( const auto& thresholdName : thresholdNames ) pProtobufTrigger->add_threshold(thresholdName); |
| 103 |
> |
for( const auto& thresholdName : thresholdNames ) pProtobufTrigger->add_varying_parameter(thresholdName); |
| 104 |
|
|
| 105 |
|
} // end of loop over triggers |
| 106 |
|
|
| 107 |
+ |
// Always make sure there is at least one Run ready to be added to |
| 108 |
+ |
std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run ); |
| 109 |
+ |
protobufRuns.push_back( std::move( pNewRun ) ); |
| 110 |
+ |
|
| 111 |
|
} |
| 112 |
|
|
| 113 |
|
l1menu::ReducedMenuSamplePrivateMembers::ReducedMenuSamplePrivateMembers( const std::string& filename ) |
| 115 |
|
{ |
| 116 |
|
GOOGLE_PROTOBUF_VERIFY_VERSION; |
| 117 |
|
|
| 118 |
< |
{ // new block to limit scope of variables, so that the file is closed immediately after reading |
| 119 |
< |
std::ifstream inputFile( filename ); |
| 120 |
< |
protobufSample.ParseFromIstream( &inputFile ); |
| 118 |
> |
// Open the file with read ability |
| 119 |
> |
int fileDescriptor = open( filename.c_str(), O_RDONLY ); |
| 120 |
> |
if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample initialise from file - couldn't open file" ); |
| 121 |
> |
::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the input file |
| 122 |
> |
google::protobuf::io::FileInputStream fileInput( fileDescriptor ); |
| 123 |
> |
google::protobuf::io::GzipInputStream gzipInput( &fileInput ); |
| 124 |
> |
google::protobuf::io::CodedInputStream codedInput( &gzipInput ); |
| 125 |
> |
|
| 126 |
> |
// First read the magic number at the start of the file and make sure it |
| 127 |
> |
// matches what I expect. As a read buffer, I'll create a string the correct |
| 128 |
> |
// size (filled with an arbitrary character) and read straight into that. |
| 129 |
> |
std::string readMagicNumber; |
| 130 |
> |
if( !codedInput.ReadString( &readMagicNumber, FILE_FORMAT_MAGIC_NUMBER.size() ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading magic number" ); |
| 131 |
> |
if( readMagicNumber!=FILE_FORMAT_MAGIC_NUMBER ) throw std::runtime_error( "ReducedMenuSample - tried to initialise with a file that is not the correct format" ); |
| 132 |
> |
|
| 133 |
> |
google::protobuf::uint32 fileformatVersion; |
| 134 |
> |
if( !codedInput.ReadVarint32( &fileformatVersion ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading file format version" ); |
| 135 |
> |
// So far I only have (and ever expect to have) one version of the file |
| 136 |
> |
// format, imaginatively versioned "1". You never know though... |
| 137 |
> |
if( fileformatVersion>1 ) std::cerr << "Warning: Attempting to read a ReducedMenuSample with version " << fileformatVersion << " with code that only knows up to version 1." << std::endl; |
| 138 |
> |
|
| 139 |
> |
google::protobuf::uint64 messageSize; |
| 140 |
> |
|
| 141 |
> |
// Read the size of the header message |
| 142 |
> |
if( !codedInput.ReadVarint64( &messageSize ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - error reading message size for header" ); |
| 143 |
> |
google::protobuf::io::CodedInputStream::Limit readLimit=codedInput.PushLimit(messageSize); |
| 144 |
> |
if( !protobufSampleHeader.ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading header" ); |
| 145 |
> |
codedInput.PopLimit(readLimit); |
| 146 |
> |
|
| 147 |
> |
// Keep looping until there is nothing more to be read from the file. |
| 148 |
> |
while( codedInput.ReadVarint64( &messageSize ) ) |
| 149 |
> |
{ |
| 150 |
> |
readLimit=codedInput.PushLimit(messageSize); |
| 151 |
> |
|
| 152 |
> |
std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run ); |
| 153 |
> |
if( !pNewRun->ParseFromCodedStream( &codedInput ) ) throw std::runtime_error( "ReducedMenuSample initialise from file - some unknown error while reading run" ); |
| 154 |
> |
protobufRuns.push_back( std::move( pNewRun ) ); |
| 155 |
> |
|
| 156 |
> |
codedInput.PopLimit(readLimit); |
| 157 |
|
} |
| 158 |
|
|
| 159 |
|
|
| 160 |
< |
// I have all of the information in the protobuf Sample, but I also need the trigger information |
| 160 |
> |
// Always make sure there is at least one Run ready to be added to. Later |
| 161 |
> |
// code assumes there is already a run there. |
| 162 |
> |
if( protobufRuns.empty() ) |
| 163 |
> |
{ |
| 164 |
> |
std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run ); |
| 165 |
> |
protobufRuns.push_back( std::move( pNewRun ) ); |
| 166 |
> |
} |
| 167 |
> |
|
| 168 |
> |
// I have all of the information in the protobuf members, but I also need the trigger information |
| 169 |
|
// in the form of l1menu::TriggerMenu. Copy out the required information. |
| 170 |
< |
for( int triggerNumber=0; triggerNumber<protobufSample.trigger_size(); ++triggerNumber ) |
| 170 |
> |
for( int triggerNumber=0; triggerNumber<protobufSampleHeader.trigger_size(); ++triggerNumber ) |
| 171 |
|
{ |
| 172 |
< |
const l1menuprotobuf::Trigger& inputTrigger=protobufSample.trigger(triggerNumber); |
| 172 |
> |
const l1menuprotobuf::Trigger& inputTrigger=protobufSampleHeader.trigger(triggerNumber); |
| 173 |
|
|
| 174 |
|
mutableTriggerMenu_.addTrigger( inputTrigger.name(), inputTrigger.version() ); |
| 175 |
|
// Get a reference to the trigger I just created |
| 209 |
|
|
| 210 |
|
void l1menu::ReducedMenuSample::addSample( const l1menu::MenuSample& originalSample ) |
| 211 |
|
{ |
| 212 |
+ |
l1menuprotobuf::Run* pCurrentRun=pImple_->protobufRuns.back().get(); |
| 213 |
+ |
|
| 214 |
|
for( size_t eventNumber=0; eventNumber<originalSample.numberOfEvents(); ++eventNumber ) |
| 215 |
|
{ |
| 216 |
+ |
// Split the events up into groups in arbitrary numbers. This is to get around |
| 217 |
+ |
// a protobuf aversion to long messages. |
| 218 |
+ |
if( pCurrentRun->event_size() > pImple_->EVENTS_PER_RUN ) |
| 219 |
+ |
{ |
| 220 |
+ |
// Gone over the arbitrary limit, so create a new protobuf Run and start |
| 221 |
+ |
// using that instead. |
| 222 |
+ |
std::unique_ptr<l1menuprotobuf::Run> pNewRun( new l1menuprotobuf::Run ); |
| 223 |
+ |
pImple_->protobufRuns.push_back( std::move( pNewRun ) ); |
| 224 |
+ |
pCurrentRun=pImple_->protobufRuns.back().get(); |
| 225 |
+ |
} |
| 226 |
+ |
|
| 227 |
|
const l1menu::IEvent& event=originalSample.getEvent( eventNumber ); |
| 228 |
< |
l1menuprotobuf::Event* pProtobufEvent=pImple_->protobufSample.add_event(); |
| 228 |
> |
l1menuprotobuf::Event* pProtobufEvent=pCurrentRun->add_event(); |
| 229 |
|
|
| 230 |
|
// Loop over all of the triggers |
| 231 |
|
for( size_t triggerNumber=0; triggerNumber<pImple_->triggerMenu.numberOfTriggers(); ++triggerNumber ) |
| 263 |
|
|
| 264 |
|
void l1menu::ReducedMenuSample::saveToFile( const std::string& filename ) const |
| 265 |
|
{ |
| 266 |
< |
std::ofstream outputFile( filename ); |
| 267 |
< |
pImple_->protobufSample.SerializeToOstream( &outputFile ); |
| 266 |
> |
// Open the file. Parameters are filename, write ability and create, rw-r--r-- permissions. |
| 267 |
> |
int fileDescriptor = open( filename.c_str(), O_WRONLY | O_CREAT, 0644 ); |
| 268 |
> |
if( fileDescriptor==0 ) throw std::runtime_error( "ReducedMenuSample save to file - couldn't open file" ); |
| 269 |
> |
::UnixFileSentry fileSentry( fileDescriptor ); // Use this as an exception safe way of closing the output file |
| 270 |
> |
|
| 271 |
> |
// Setup the protobuf file handlers |
| 272 |
> |
google::protobuf::io::FileOutputStream fileOutput( fileDescriptor ); |
| 273 |
> |
google::protobuf::io::GzipOutputStream gzipOutput( &fileOutput ); |
| 274 |
> |
google::protobuf::io::CodedOutputStream codedOutput( &gzipOutput ); |
| 275 |
> |
|
| 276 |
> |
// Write a magic number at the start of all files |
| 277 |
> |
codedOutput.WriteString( pImple_->FILE_FORMAT_MAGIC_NUMBER ); |
| 278 |
> |
// Write an integer that specifies what version of the file format I'm using. I |
| 279 |
> |
// have no intention of changing the format but I might as well keep the option |
| 280 |
> |
// open. |
| 281 |
> |
codedOutput.WriteVarint32( 1 ); |
| 282 |
> |
|
| 283 |
> |
// Write the size of the header message into the file... |
| 284 |
> |
codedOutput.WriteVarint64( pImple_->protobufSampleHeader.ByteSize() ); |
| 285 |
> |
// ...and then write the header |
| 286 |
> |
pImple_->protobufSampleHeader.SerializeToCodedStream( &codedOutput ); |
| 287 |
> |
|
| 288 |
> |
// Now go through each of the runs and do the same for those |
| 289 |
> |
for( const auto& pRun : pImple_->protobufRuns ) |
| 290 |
> |
{ |
| 291 |
> |
codedOutput.WriteVarint64( pRun->ByteSize() ); |
| 292 |
> |
pRun->SerializeToCodedStream( &codedOutput ); |
| 293 |
> |
} |
| 294 |
> |
|
| 295 |
|
} |
| 296 |
|
|
| 297 |
|
size_t l1menu::ReducedMenuSample::numberOfEvents() const |
| 298 |
|
{ |
| 299 |
< |
return pImple_->protobufSample.event_size(); |
| 299 |
> |
size_t numberOfEvents=0; |
| 300 |
> |
for( const auto& pRun : pImple_->protobufRuns ) numberOfEvents+=pRun->event_size(); |
| 301 |
> |
return numberOfEvents; |
| 302 |
|
} |
| 303 |
|
|
| 304 |
|
const l1menu::IReducedEvent& l1menu::ReducedMenuSample::getEvent( size_t eventNumber ) const |
| 305 |
|
{ |
| 306 |
< |
pImple_->event.pProtobufEvent=pImple_->protobufSample.mutable_event(eventNumber); |
| 307 |
< |
return pImple_->event; |
| 306 |
> |
for( const auto& pRun : pImple_->protobufRuns ) |
| 307 |
> |
{ |
| 308 |
> |
if( eventNumber<static_cast<size_t>(pRun->event_size()) ) |
| 309 |
> |
{ |
| 310 |
> |
pImple_->event.pProtobufEvent=pRun->mutable_event(eventNumber); |
| 311 |
> |
return pImple_->event; |
| 312 |
> |
} |
| 313 |
> |
// Event must be in a later run, so reduce the number by how many events |
| 314 |
> |
// were in this run and look again. |
| 315 |
> |
eventNumber-=pRun->event_size(); |
| 316 |
> |
} |
| 317 |
> |
|
| 318 |
> |
// Should always find the event before getting to this point, so throw an |
| 319 |
> |
// exception if this happens. |
| 320 |
> |
throw std::runtime_error( "ReducedMenuSample::getEvent(eventNumber) was asked for an invalid eventNumber" ); |
| 321 |
|
} |
| 322 |
|
|
| 323 |
|
const l1menu::TriggerMenu& l1menu::ReducedMenuSample::getTriggerMenu() const |
