From ead3a7739afbfe062be611f9b3ca125178ff84b0 Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Tue, 12 Jul 2022 15:47:27 +0200 Subject: [PATCH 01/10] Initial implementation of converter bin to SONATA reports - Read .bbp with reportinglib ReadManager - Write using write_buffered_data() --- CMakeLists.txt | 9 + converter/CMakeLists.txt | 36 + converter/binary_reader/BinaryPositions.h | 62 ++ converter/binary_reader/CMakeLists.txt | 3 + converter/binary_reader/ReadManager.cpp | 886 ++++++++++++++++++ converter/binary_reader/ReadManager.h | 722 ++++++++++++++ .../binary_reader/crossPlatformConversion.h | 28 + converter/reports_converter.cpp | 73 ++ converter/spikes_converter.cpp | 52 + include/bbp/sonata/reports.h | 9 + src/data/sonata_data.cpp | 22 +- src/data/sonata_data.h | 3 +- src/library/report.cpp | 8 +- src/library/report.h | 2 + src/library/reports.cpp | 16 + tests/integration/integration_test.cpp | 33 +- 16 files changed, 1949 insertions(+), 15 deletions(-) create mode 100644 converter/CMakeLists.txt create mode 100644 converter/binary_reader/BinaryPositions.h create mode 100644 converter/binary_reader/CMakeLists.txt create mode 100644 converter/binary_reader/ReadManager.cpp create mode 100644 converter/binary_reader/ReadManager.h create mode 100644 converter/binary_reader/crossPlatformConversion.h create mode 100644 converter/reports_converter.cpp create mode 100644 converter/spikes_converter.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 481d376..b249365 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,8 @@ option(SONATA_REPORT_ENABLE_SUBMODULES "Use Git submodules for header-only depen option(SONATA_REPORT_ENABLE_WARNING_AS_ERROR "Compile C++ with warnings as errors" OFF) option(SONATA_REPORT_ENABLE_MPI "Enable MPI-based execution" ON) option(SONATA_REPORT_ENABLE_TEST "Enable tests" ON) +option(SONATA_REPORT_ENABLE_CONVERTER "Enable binary report converter" OFF) + set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake) set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true) @@ -157,3 +159,10 @@ if(SONATA_REPORT_ENABLE_TEST) ) endif() endif() + +# ============================================================================= +# Converter of binary to SONATA reports +# ============================================================================= +if(SONATA_REPORT_ENABLE_CONVERTER) + add_subdirectory(converter) +endif() diff --git a/converter/CMakeLists.txt b/converter/CMakeLists.txt new file mode 100644 index 0000000..1b2a771 --- /dev/null +++ b/converter/CMakeLists.txt @@ -0,0 +1,36 @@ +add_subdirectory(binary_reader) +# Convert binary reports to SONATA format +add_executable(spikes_converter spikes_converter.cpp) +target_include_directories(spikes_converter + PRIVATE + $ + $ + SYSTEM + ${PROJECT_SOURCE_DIR}/extlib/spdlog/include +) +target_compile_options(spikes_converter + PRIVATE ${SONATA_REPORT_COMPILE_OPTIONS} +) +target_link_libraries(spikes_converter + PUBLIC sonata_report + PRIVATE spdlog::spdlog_header_only + PRIVATE ${MPI_CXX_LIBRARIES} +) + +add_executable(reports_converter reports_converter.cpp) +target_include_directories(reports_converter + PRIVATE + $ + $ + SYSTEM + ${PROJECT_SOURCE_DIR}/extlib/spdlog/include +) +target_compile_options(reports_converter + PRIVATE ${SONATA_REPORT_COMPILE_OPTIONS} +) +target_link_libraries(reports_converter + PUBLIC sonata_report + PUBLIC binary_reader + PRIVATE spdlog::spdlog_header_only + PRIVATE ${MPI_CXX_LIBRARIES} +) \ No newline at end of file diff --git a/converter/binary_reader/BinaryPositions.h b/converter/binary_reader/BinaryPositions.h new file mode 100644 index 0000000..28a7a9d --- /dev/null +++ b/converter/binary_reader/BinaryPositions.h @@ -0,0 +1,62 @@ +// FixedPositions // Don't change so we can have backward compatibility + +// If identifier read at position 0 matches ARCHITECTURE_IDENTIFIER, then the file was writting from +// native architecture +#define ARCHITECTURE_IDENTIFIER 1.001 + +// Double: +#define IDENTIFIER_POSITION 0 +// Int: +#define HEADER_SIZE_POSITION IDENTIFIER_POSITION + sizeof(double) +// String: +#define LIBRARY_VERSION_POSITION 16 +// String: +#define SIMULATOR_VERSION_POSITION 32 +// Int: +#define TOTAL_NUMBER_OF_CELLS_POSITION 48 +// Int: +#define TOTAL_NUMBER_OF_COMPARTMENTS_POSITION 52 +// Int: +#define NUMBER_OF_STEPS_POSITION 64 +// Double: +#define TIME_START_POSITION 72 +// Double: +#define TIME_END_POSITION 80 +// Double: +#define DT_TIME_POSITION 88 +// String: +#define D_UNIT_POSITION 96 +// String: +#define T_UNIT_POSITION 112 +// Int: +#define MAPPING_SIZE_POSITION 128 +// String: +#define MAPPING_NAME_POSITION 144 +// Int: +#define EXTRA_MAPPING_SIZE_POSITION 160 +// String: +#define EXTRA_MAPPING_NAME_POSITION 176 +// String: +#define TARGET_NAME_POSITION 192 + +// Length definition +// int: +#define HEADER_LENGTH 1024 +// int: +#define SIZE_CELL_INFO_LENGTH 64 +// int: +#define SIZE_STRING 16 + +// Inside CellInfo +// Int: +#define NUMBER_OF_CELL_POSITION 0 +// Int: +#define NUMBER_OF_COMPARTMENTS_POSITION 8 +// OffSet: +#define DATA_INFO_POSITION 16 +// OffSet: +#define EXTRA_MAPPING_INFO_POSITION 24 +// OffSet: +#define MAPPING_INFO_POSITION 32 + +// order of the cell is a simple string order diff --git a/converter/binary_reader/CMakeLists.txt b/converter/binary_reader/CMakeLists.txt new file mode 100644 index 0000000..61ab63f --- /dev/null +++ b/converter/binary_reader/CMakeLists.txt @@ -0,0 +1,3 @@ + +set(READER_SOURCES ReadManager.cpp) +add_library (binary_reader STATIC ${READER_SOURCES}) diff --git a/converter/binary_reader/ReadManager.cpp b/converter/binary_reader/ReadManager.cpp new file mode 100644 index 0000000..d44cbb5 --- /dev/null +++ b/converter/binary_reader/ReadManager.cpp @@ -0,0 +1,886 @@ +#include +#include +#include "ReadManager.h" + +using namespace std; +namespace bbpReader { + + Header::Header(char* readBuffer) { +#define fetch(TYPE, POSITION) *((TYPE*)(readBuffer + POSITION)) +#define fetchS(POSITION) string((char*)(readBuffer + POSITION)) + + double readIdentifier = fetch(double, IDENTIFIER_POSITION); + headerSize = fetch(int32_t, HEADER_SIZE_POSITION); + amountOfCells = fetch(int32_t, TOTAL_NUMBER_OF_CELLS_POSITION); + /** \bug 32 bit is consistent with the binary format specification of the + header, however + */ + uint32_t compartmentsPerFrame = fetch(int32_t, TOTAL_NUMBER_OF_COMPARTMENTS_POSITION); + libraryVersion = fetchS(LIBRARY_VERSION_POSITION); + simulatorVersion = fetchS(SIMULATOR_VERSION_POSITION); + numberOfSteps = fetch(int32_t, NUMBER_OF_STEPS_POSITION); + startTime = fetch(double, TIME_START_POSITION); + endTime = fetch(double, TIME_END_POSITION); + dt = fetch(double, DT_TIME_POSITION); + dataUnit = fetchS(D_UNIT_POSITION); + timeUnit = fetchS(T_UNIT_POSITION); + mappingSize = fetch(int32_t, MAPPING_SIZE_POSITION); + mappingName = fetchS(MAPPING_NAME_POSITION); + extraMappingSize = fetch(int32_t, EXTRA_MAPPING_SIZE_POSITION); + extraMappingName = fetchS(EXTRA_MAPPING_NAME_POSITION); + targetName = fetchS(TARGET_NAME_POSITION); + +#undef fetchS +#undef fetch + + isNative = readIdentifier == ARCHITECTURE_IDENTIFIER; + + if (!isNative) { + // check if data can be converted: + DOUBLESWAP(readIdentifier); + if (readIdentifier != ARCHITECTURE_IDENTIFIER) { + cerr << "File is corrupt or originated from an unknown architecture." << endl; + exit(1); + } + + // if data was stored on a machine with different architecture, + // then the data needs to be converted to be compatible with the reading machine + FIX_INT(headerSize); + FIX_INT(amountOfCells); + FIX_INT(compartmentsPerFrame); + FIX_INT(numberOfSteps); + DOUBLESWAP(startTime); + DOUBLESWAP(endTime); + DOUBLESWAP(dt); + FIX_INT(mappingSize); + FIX_INT(extraMappingSize); + } + + // printf( "read %d compartment count\n", compartmentsPerFrame ); + totalAmountOfCompartments = compartmentsPerFrame; + } + + bool Header::operator==(const Header& h1) const { +#define _CHECK(field) \ + if (h1.field != field) \ + return false + // compare all fields other than isNative + _CHECK(headerSize); + _CHECK(libraryVersion); + _CHECK(simulatorVersion); + _CHECK(amountOfCells); + _CHECK(totalAmountOfCompartments); + _CHECK(numberOfSteps); + _CHECK(startTime); + _CHECK(endTime); + _CHECK(dt); + _CHECK(dataUnit); + _CHECK(timeUnit); + _CHECK(mappingSize); + _CHECK(mappingName); + _CHECK(extraMappingName); + _CHECK(extraMappingSize); + _CHECK(targetName); + + // else if nothing returned false: + return true; +#undef _CHECK + } + + CellInfo::CellInfo() { + cellNum = 0; + master = &cellNum; + amountOfCompartments = 0; + dataLocation = 0; + extraMappingLocation = 0; + mappingLocation = 0; + } + + CellInfo::CellInfo(char* readBuffer, bool isNative, key cmp) { +#define fetch(TYPE, POSITION) *((TYPE*)(readBuffer + POSITION)) + cellNum = fetch(int32_t, NUMBER_OF_CELL_POSITION); + amountOfCompartments = fetch(int32_t, NUMBER_OF_COMPARTMENTS_POSITION); + + dataLocation = fetch(uint64_t, DATA_INFO_POSITION); + extraMappingLocation = fetch(uint64_t, EXTRA_MAPPING_INFO_POSITION); + mappingLocation = fetch(uint64_t, MAPPING_INFO_POSITION); + +#undef fetch + + if (!isNative) { + // cout << "Fixing cell info data..." << endl; + FIX_INT(cellNum); + FIX_INT(amountOfCompartments); + + // cout << "size of offset type is " << sizeof(dataLocation) << endl; + DOUBLESWAP(dataLocation); + DOUBLESWAP(extraMappingLocation); + DOUBLESWAP(mappingLocation); + } + + // fprintf( stderr, "build cell info: gid %d dataPos %d extraPos %d mappingPos %d\n", + // cellNum, (int) dataLocation, (int) extraMappingLocation, (int) mappingLocation ); + // cerr<<"build cell info: gid "<) + DEFINE_CMP_OP(>=) + +#undef DEFINE_CMP_OP +#undef COMPARE_ + + bool CellInfo::compareByOffset(const CellInfo& a, const CellInfo& b) { + return a.dataLocation < b.dataLocation; + } + + FrameInfo::FrameInfo(char* buffer, char* endOfBuffer, bool isNative, CellInfo::key sortMode) { + // create a cellinfo and then insert into itself (rem, this class inherits from set). The + // resulting set iterator is then put into + // a map member for associative lookup gid:location + for (char* bufferLoc = buffer; bufferLoc < endOfBuffer; + bufferLoc += SIZE_CELL_INFO_LENGTH) { + iterator ins = insert(CellInfo(bufferLoc, isNative, sortMode)).first; + gidLookup.insert(make_pair(ins->getCellNum(), ins)); + } + } + + FrameInfo::FrameInfo(const FrameInfo& rhs) : baseTypes::FrameInfo() { + *this = rhs; + } + + FrameInfo& FrameInfo::operator=(const FrameInfo& rhs) { + insert(rhs.begin(), rhs.end()); + for (iterator cellItem = begin(); cellItem != end(); ++cellItem) { + gidLookup.insert(make_pair(cellItem->getCellNum(), cellItem)); + } + + return *this; + } + + FrameInfo& FrameInfo::filterGIDs(const vector& gids) { + Lookup newGidLookup; + + for (vector::const_iterator gid = gids.begin(); gid != gids.end(); gid++) { + Lookup::iterator it = gidLookup.find(*gid); + if (it != gidLookup.end()) { + newGidLookup.insert(*it); + gidLookup.erase(it); + } else + throw string( + "GID doesn't exist in file."); // TODO: replace by custom std::exception + } + + // gidLookup contains the gids that need to be removed and the newGidLookup contains the + // gids that remain. + + for (Lookup::iterator i = gidLookup.begin(); i != gidLookup.end(); i++) + erase(i->second); // delete the unwanted gids. + + gidLookup.swap(newGidLookup); + + return *this; + } + + CompartmentMapping::CompartmentMapping() : Base() { + } + + CompartmentMapping::CompartmentMapping(MappingItem* start, MappingItem* _end) + : Base(start, _end) { + } + + CompartmentMapping::CompartmentMapping(MappingItem* buffer, SizeType bufSize) + : Base(buffer, buffer + bufSize) { + } + + SectionData::SectionData() : Base() { + } + + SectionData::SectionData(DataItem firstData) { + push_back(firstData); + } + + SectionData::SectionData(DataItem* start, DataItem* _end) : Base(start, _end) { + } + + bool SectionData::operator==(const SectionData& s1) { + if (s1.size() != size()) + return false; + + const_iterator dat1, dat2; + for (dat1 = s1.begin(), dat2 = begin(); dat2 != end(); dat1++, dat2++) + if (*dat1 != *dat2) + return false; + + return true; + } + + bool SectionData::operator!=(const SectionData& s1) { + return !(*this == s1); + } + + bool SectionData::compareByOriginalLocation(const Identifier& c1, const Identifier& c2) { + return c1.getLocation() < c2.getLocation(); + } + + CellData::CellData() { + sectSort = SID; + } + + CellData::CellData(SortingKey sorting) { + sectSort = sorting; + } + + CellData::CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { + sectSort = SID; + fillData(dataBuffer, mapping); + } + + void CellData::fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { + CompartmentMapping::const_iterator it; + SectionIndex i; + for (it = mapping.begin(), i = 0; it != mapping.end(); it++, dataBuffer++, i++) { + // iteration through all compartments inside cell + DataItem data = *dataBuffer; + + iterator segMatch = find(*it); // find corresponding segment + + if (segMatch == end()) { + // SID was not found. + // this means it wasnt created yet. + // therefore create a new one and insert first data item into it: + iterator ins = + insert(make_pair(SectionData::Identifier(*it, i, sectSort), SectionData(data))) + .first; + // note that it is guaranteed that the above insertion takes place, because this + // block is only entered + // if the sid does not exist inside this CellData object yet. + // if sorting is done by location, this will be unique anyways. + sidLookup.insert(make_pair(*it, ins)); + } else + // SID is already present in CellData. The new data item just needs to be inserted + segMatch->second.push_back(data); + } + } + + // typedef CellDataImpl UnsortedCellData; + + bool CellData::operator==(const CellData& d1) { + if (d1.size() != size()) + return false; + + const_iterator seg1, seg2; + + for (seg1 = d1.begin(), seg2 = begin(); seg2 != end(); seg1++, seg2++) { + if (seg1->first != seg2->first) + return false; + if (seg1->second != seg2->second) + return false; + } + + return true; + } + + ReadManager::ReadManager() { + } + + ReadManager::ReadManager(const string& filePath) + : ifstream(filePath.c_str(), ios::in | ios::binary) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); + // open(filePath); + readHeader(); + } + + void ReadManager::open(const string& filePath) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); + + ifstream::open(filePath.c_str(), ios::in | ios::binary); + readHeader(); + } + + void ReadManager::readHeader() { + char readBuffer[HEADER_LENGTH + 1]; + + seekg(ios::beg); + + ifstream::read(readBuffer, HEADER_LENGTH); + + readBuffer[HEADER_LENGTH] = 0; // c string terminator + fileInfo = readBuffer; + + cellInfoOrigin = tellg(); + + isNative = fileInfo.sourceIsNative(); + + CellInfo firstCell = retrieveCellInfo(0); + + startLocation = firstCell.getDataLocation(); + mappingLocation = firstCell.getMappingLocation(); + + mappingSize_floats = fileInfo.getMappingSize() * fileInfo.getTotalNumberOfCompartments(); + } + + CellInfo ReadManager::makeNullInfo() { + char fakeBuffer[SIZE_CELL_INFO_LENGTH]; + + for (char *i = fakeBuffer, *bufferEnd = fakeBuffer + SIZE_CELL_INFO_LENGTH; i < bufferEnd; + i++) + *i = 0; + + return CellInfo(fakeBuffer); + } + + const CellInfo ReadManager::_null_info = ReadManager::makeNullInfo(); + + CellInfo ReadManager::retrieveFindCell(CellID cellNum) { + int start = (cellNum > fileInfo.getNumberOfCells()) + ? fileInfo.getNumberOfCells() - 1 + : cellNum - 1; // in many cases cellNum will be same as cellIndex + CellInfo seek; + + bool oppositeSearch = false; + + int i = start; + do { + if (i >= (int)fileInfo.getNumberOfCells() || i < 0) { + if (oppositeSearch) + return makeNullInfo(); + else { + oppositeSearch = true; + i = start; + } + } + + seek = retrieveCellInfo(i); + + if ((int)seek.getCellNum() < i) + if (oppositeSearch) + i--; + else + i++; + else if (oppositeSearch) + i++; + else + i--; + + } while (seek.getCellNum() != cellNum); + + return seek; + } + + CellInfo ReadManager::retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode) { + FilePosition ppos = tellg(); + + seekgCellInfo(cellIndex); + + char readBuffer[SIZE_CELL_INFO_LENGTH]; + read(readBuffer, SIZE_CELL_INFO_LENGTH); + + CellInfo reqInfo(readBuffer, isNative, sortMode); + + seekg(ppos); + + return reqInfo; + } + + FrameInfo ReadManager::retrieveAllCellInfo(CellInfo::key sortMode) { + FilePosition ppos = tellg(); + + CellCount amountOfCells = fileInfo.getNumberOfCells(); + streamsize bytesToRead = SIZE_CELL_INFO_LENGTH * amountOfCells; + char* readBuffer = new char[bytesToRead]; + char* endOfBuffer = readBuffer + bytesToRead; + seekgCellInfo(0); + + read(readBuffer, bytesToRead); + + FrameInfo allCells(readBuffer, endOfBuffer, isNative, sortMode); + + delete[] readBuffer; + + seekg(ppos); + + return allCells; + } + + void ReadManager::readFrame(DataItem* dataBuffer) { + read(dataBuffer, fileInfo.getTotalNumberOfCompartments()); + } + + void ReadManager::readFrameMapping(MappingItem* buffer) { + FilePosition ppos = tellg(); + + seekgMappingStart(); + + read(buffer, mappingSize_floats); + + seekg(ppos); + } + + FrameParser::FrameOrganiser::FrameOrganiser() { + sectSort = SID; + } + + FrameParser::FrameOrganiser::FrameOrganiser(SortingKey segmentSorting) { + sectSort = segmentSorting; + } + + void FrameParser::FrameOrganiser::fillData(const FrameInfo& info, + const vector& mapping, + const DataItem* dataBuffer) { + FrameInfo::const_iterator pinf = info.begin(); + vector::const_iterator pmap = mapping.begin(); + + for (const DataItem *ptr = dataBuffer; pinf != info.end(); + ptr += pinf->getAmountOfCompartments(), pinf++, pmap++) { + // iteration through all gids + + iterator ins = insert(make_pair(*pinf, CellData(sectSort))) + .first; // insert CellData object for current gid + ins->second.fillData(ptr, *pmap); // feed new CellData object with mapping and data + } + } + + void FrameParser::FrameOrganiser::makeOrderedDataBuffer(DataItem* buffer) const { + for (const_iterator cd = begin(); cd != end(); cd++) // iterate through cells + for (CellData::const_iterator sd = cd->second.begin(); sd != cd->second.end(); + sd++) // iterate through segments + for (SectionData::const_iterator f = sd->second.begin(); f != sd->second.end(); + f++, buffer++) // iterate through compartments + *buffer = *f; + } + + FrameParser::CompartmentIndexing FrameParser::FrameOrganiser::makeOrderedOffsetReferences() + const { + CompartmentIndexing gidV; + gidV.reserve(size()); + + FrameDataIndex position = 0; + + for (const_iterator gid = begin(); gid != end(); gid++) { // iterate through gids + gidV.push_back(vector()); + + // reserve memory for amount of segments + gidV.back().reserve((int)(gid->second.size() * 1.5)); + + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); + position += sid->second.size(), sid++) { + // iterating through available sid's + while (presid++ != sid->first.getSID()) + gidV.back().push_back(0); // insert dummy variables for unavailable sids (gaps) + gidV.back().push_back(position); + } + } + + return gidV; + } + + FrameParser::CompartmentCounts FrameParser::FrameOrganiser::makeOrderedSegSizes() const { + CompartmentCounts gidV; + gidV.reserve(size()); + + for (const_iterator gid = begin(); gid != end(); gid++) { + gidV.push_back(vector()); + gidV.back().reserve((int)(gid->second.size() * 1.5)); + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); + sid++) { + while (presid++ != sid->first.getSID()) + gidV.back().push_back(0); // fill sid gaps with zero sizes + gidV.back().push_back(sid->second.size()); + } + } + + return gidV; + } + + FrameParser::FrameParser() { + readBuffer = NULL; + refArray = NULL; + } + + FrameParser::FrameParser(const string& file, bool sortData) : ReadManager(file) { + if (fileInfo.getMappingSize() != 1) { + cerr + << "Target file has incompatible mapping size for FrameParser. FrameParser only works with mapping size 1." + << endl; + exit(1); + } + + CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; + + FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); + + ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); + offsets = ReadOptimiser(fileInfo.getTotalNumberOfCompartments()); // * sizeof(DataItem) ); + createReferences(cinfo, sortData); + + timestep = 0; // by default start pointing to timstep 0 + ReadManager::seekgFrame(timestep); + } + + FrameParser::FrameParser(const string& file, const vector& gidTarget, bool sortData) + : ReadManager(file) { + timestep = 0; + ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); + retarget(gidTarget, sortData); + } + + FrameParser::~FrameParser() { + if (readBuffer) // readBuffer may have not been used if sorting was disabled, in which case + // it would have been set to NULL + delete[] readBuffer; + if (refArray) // same applies for refArray + delete[] refArray; + } + + void FrameParser::retarget(const vector& gidTarget, bool sortData) { + CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; + + FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); + cinfo.filterGIDs(gidTarget); + + offsets = ReadOptimiser(cinfo, getFrameSize_bytes(), getStartOffset()); + offsets.setElementSize(sizeof(DataItem)); + + createReferences(cinfo, sortData); + + seekg(getStartOffset() + offsets.getFirst()); + + operator=(timestep); + } + + void FrameParser::createReferences(const FrameInfo& cinfo, bool sortData) { + SortingKey sSorting = (sortData) ? SID : LOC; + + refArray = NULL; + readBuffer = NULL; + + vector allMapping; + + DataItem* tempBuffer = + new DataItem[offsets.getDataSize_elements()]; // allocate memory for readBuffer. This + // will atleast be needed for creating + // the FrameOrganiser + + readFrameMapping(tempBuffer); + + FrameInfo::const_iterator pinf = cinfo.begin(); + + for (DataItem *mpptr = tempBuffer, *endptr = mpptr + pinf->getAmountOfCompartments(); + pinf != cinfo.end(); pinf++, mpptr = endptr, endptr += pinf->getAmountOfCompartments()) + allMapping.push_back(CompartmentMapping(mpptr, endptr)); + + if (sortData) + for (FrameDataIndex i = 0; i < offsets.getDataSize_elements(); i++) + tempBuffer[i] = i; // makes buffer of indexes { 0 , 1 ,2 , 3 , ... } + // else it doesnt really matter what FrameOrganiser is fed with. + + org = FrameOrganiser(sSorting); + + org.fillData(cinfo, allMapping, tempBuffer); // feed the FrameOrganiser with the data. + + // references = org.makeOrderedOffsetReferences(); + // segSizes = org.makeOrderedSegSizes(); + + if (sortData) { + readBuffer = tempBuffer; // keep allocated memory at tempBuffer + + // sortData is enabled -> create an array of pointers to readBuffer + // in which the pointers are sorted by GID and SID of the data the point to. + // these pointers are to be stored into ref array + org.makeOrderedDataBuffer( + readBuffer); // rearranges indexes to correspond to gid and segment order + // the memory allocated for readBuffer is now being used for a different purpose: + // it contains the sorted indexes, which now need to be converted to pointers to + // readBuffer. + + refArray = new DataItem*[offsets.getDataSize_elements()]; // allocate memory for + // refArray (will contain + // pointers for sorting) + + for (float *pIndex = readBuffer, **pPtr = refArray, + *pEnd = readBuffer + offsets.getDataSize_elements(); + pIndex != pEnd; pIndex++, pPtr++) + *pPtr = &readBuffer[(FrameDataIndex)*pIndex]; // converts indexes to pointers and + // stores them into refArray + } else + // data sorting is disabled -> allocated memory at temp Buffer is no longer needed. + //(data will be read directly into the memory that user has provided) + delete[] tempBuffer; + } + + void FrameParser::bufferTransfer(DataItem* buffer) const { + DataItem* reqLocation = buffer; + + if (buffer == readBuffer) // source and destination is the same - trouble! + buffer = new DataItem[offsets.getDataSize_elements()]; // need to allocate different + // destination + + for (DataItem **sourcePtr = refArray, *target = buffer, + *dEnd = buffer + offsets.getDataSize_elements(); + target != dEnd; sourcePtr++, target++) + *target = **sourcePtr; + + if (reqLocation != buffer) { // buffer has not been transfered where requested! + // this can now be corrected. + for (DataItem *origin = buffer, *destination = reqLocation, + *dEnd = buffer + offsets.getDataSize_elements(); + origin != dEnd; origin++, destination++) + *destination = *origin; // copy data to requested location + delete[] buffer; // buffer was is only a temporary memory allocation - delete it. + } + } + + FrameParser& FrameParser::operator++() { + timestep++; + seekg(fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; + } + + FrameParser& FrameParser::operator++(int) { + return this->operator++(); + } + + FrameParser& FrameParser::operator--() { + timestep--; + seekg(-fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; + } + + FrameParser& FrameParser::operator--(int) { + return this->operator--(); + } + + FrameParser& FrameParser::operator+=(FrameDiff increment) { + timestep += increment; + seekg(increment * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; + } + + FrameParser& FrameParser::operator-=(FrameDiff decrement) { + timestep -= decrement; + seekg(-decrement * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; + } + + FrameParser& FrameParser::operator=(FrameIndex newTime) { + timestep = newTime; + seekgFrame(timestep); + seekg(offsets.getFirst(), cur); + return *this; + } + + Time FrameParser::getTime() const { + return fileInfo.getStartTime() + timestep * fileInfo.getTimeStepSize(); + } + + void FrameParser::readFrame(DataItem* buffer) { + DataItem* readPtr; + + if (readBuffer) // readBuffer will be NULL if sorting is disabled + readPtr = readBuffer; // sorting enabled -> read into readBuffer and transfer data in + // order later + else + readPtr = buffer; // sorting disabled -> read directly into buffer + + // offsets contains alternating values of data sizes to be read and skipped. + for (ReadOptimiser::iterator i = offsets.begin(); i != offsets.end(); i++) { + read(readPtr, *i); + readPtr += *i; + i++; + + // seems to be confusion as to whether offsets should contain bytelength or element + // count. I have it contain element count for now + // i now points to an offset that indicates amount of data to be skipped + if (i == offsets.end()) // no more offsets -> i'm done reading + break; + + if (*i) + seekg(*i, ios::cur); // non-zero offset -> perfom the skip + } + + if (refArray) // refArray will be NULL if sorting is disabled + bufferTransfer(buffer); // sorting is enabled. Transfer data from readBuffer to buffer + // such that data is in order + } + + void FrameParser::readFrameMapping(MappingItem* buffer) { + // readFrameMapping should not return without the filepointer being in the same state as + // before it was called. + FilePosition before = tellg(); // make a backup for current filepointer + + seekgMappingStart(); // now move file pointer to mapping location + + size_t prevElement = offsets.getElementSize(); // backup element size. + offsets.setElementSize(sizeof(MappingItem)); // setup offsets to be reading MappingItems + // instead of whatever it was reading before. + + seekg(offsets.getFirst(), ios::cur); // and advance to first cell + readFrame( + buffer); // read the data - this function will also alter the filepointer and timestep + + offsets.setElementSize(prevElement); // restore element size. + + seekg(before); // restore filepointer + } + + void FrameParser::readFrameData(DataItem* buffer) { + readFrame(buffer); + timestep++; // the filepointer has been incremented to the next timestep as data has been + // read. + } + + size_t FrameParser::ReadOptimiser::defaultElementSize = 1; + + FrameParser::ReadOptimiser::ReadOptimiser() { + elementSize = 0; + firstOffset = 0; + dataSize_elements = 0; + } + + FrameParser::ReadOptimiser::ReadOptimiser(std::streamsize frameSize_bytes) { + elementSize = defaultElementSize; + firstOffset = 0; + dataSize_elements = frameSize_bytes; // * elementSize; + push_back(dataSize_elements); + } + + FrameParser::ReadOptimiser::ReadOptimiser(const FrameInfo& cellsToRead, + std::streamsize frameSize_bytes, + FilePosition startLocation) { + elementSize = defaultElementSize; + // collect data edge points + multiset dataEdges; + + dataEdges.insert(startLocation); + + for (FrameInfo::const_iterator i = cellsToRead.begin(); i != cellsToRead.end(); i++) { + dataEdges.insert(i->getDataLocation()); // starting point + dataEdges.insert(i->getDataLocation() + + i->getAmountOfCompartments() * elementSize); // end point + } + + dataEdges.insert(startLocation + (FileOffset)frameSize_bytes); + + /*create a vector that contains the offsets. + * The first offset represents the first bunch of data to be read. + * The second offset then represents the amound of data to be skipped after the first read. + * and then the next offset is again the amount of data to be read. + */ + + // compute the offsets differentiating the dataEdges + list offsets; + for (multiset::iterator currentPos = dataEdges.begin(), + prevPos = currentPos++; + currentPos != dataEdges.end(); prevPos = currentPos, currentPos++) + offsets.push_back(*currentPos - *prevPos); + + // the last offsets should move the file pointer beyond the frame to the start location on + // the next frame + // therefore the first and last offsets are special -> take them out + + firstOffset = offsets.front(); + FileOffset lastOffset = offsets.back(); + + offsets.pop_front(); // remove first offset from list + offsets.pop_back(); // remove last offset from list + + // now the first an last offset are read dimensions. This guarantees that a skip has always + // a precessor and successor + // this is convenient in the next step, as a skip is always on an even location aswell as + // offsets.end() + + list::iterator it; + bool toggleOnRead; + + dataSize_elements = 0; + for (toggleOnRead = true, it = offsets.begin(); it != offsets.end(); + toggleOnRead = !toggleOnRead, it++) { + if (toggleOnRead) { + dataSize_elements += (*it /= elementSize); + // handle read + // read specifications should be in units of floats, but skips should be in units of + // bytes. + // also size of data to be read per frame needs to established + } else + // handle skip - There should be no zero offset inbetween + if (*it == 0) { + // remove this zero skip by combining prev. and next read. + list::iterator prevRead = it; + list::iterator nextRead = it; + prevRead--; + nextRead++; + + dataSize_elements += + (*nextRead /= elementSize); // next read was still in units of bytes + *prevRead += *nextRead; // combine nextRead with prevRead + offsets.erase(it); + offsets.erase(nextRead); + it = prevRead; // point to prevRead now + toggleOnRead = + true; // and mark that toggle flag that current iteration deal with read. + } + } + + // now optimise access performance by converting the offsets list to a vector. + + vector voffsets(offsets.begin(), offsets.end()); + swap(voffsets); + + // compute the finalSkip, which brings the filePointer back to the starting location, but of + // the nextFrame + FileOffset finalSkip = lastOffset + firstOffset; + + if (finalSkip) + push_back(finalSkip); + } + + void FrameParser::ReadOptimiser::updateOffsets(size_t newSize) { + dataSize_elements = newSize * dataSize_elements / elementSize; + for (iterator i = begin(); i != end(); i++) { // iterate through all read offsets + // first reduce the read offsets to byte numbers, then convert it to the new element + // size + *i = newSize * (*i) / elementSize; + i++; + if (i == end()) + break; + } + } +} diff --git a/converter/binary_reader/ReadManager.h b/converter/binary_reader/ReadManager.h new file mode 100644 index 0000000..97abe37 --- /dev/null +++ b/converter/binary_reader/ReadManager.h @@ -0,0 +1,722 @@ +#ifndef _ReadManager__ +#define _ReadManager__ + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "BinaryPositions.h" + +#include "crossPlatformConversion.h" + +/*Terminology: + * Compartment: lowest subdivision of cell. (cannot be broken down further) + * Section: group of unbranched compartments (may sometimes be called segment, please change to + * section if found) + * Frame: Data residing in a single timestep. The scope of a frame may vary in terms of which cells + * it includes, depending on its use. + * See documentation for further naming conventions. + */ + +namespace bbpReader { + + // class list: + class Header; + class CellInfo; + class FrameInfo; + class CompartmentMapping; + class SectionData; + // class SectionData::Identifier; + class Identifier; + class CellData; + class ReadManager; + class FrameParser; + // class FrameParser::FrameOrganiser; + // class FrameParser::ReadOptimiser; + // end class list + + // type conventions: + + typedef uint32_t CellCount; // indications amount of cells. + typedef uint32_t CellID; // used for GIDs + typedef uint32_t CellIndex; // used for indexing arrays of cells.s + typedef uint32_t FrameCount; // indictions of amount of timesteps. + typedef uint32_t FrameIndex; // used for frame numbers + typedef int32_t + FrameDiff; // used for indicating time periods in terms of frame indexes (can be negative) + typedef double Time; // used for indicating biological time durations. + typedef uint64_t FrameDataCount; // used for indicating amount of data items for an entire + // frame + typedef uint64_t FrameDataIndex; // used for indexing arrays, vectors etc of data items. + typedef uint32_t CellDataCount; // used for indicating amount of data items that reside inside + // a single Cell. + typedef uint16_t SectionDataCount; // used for indicating amount of data items that reside + // inside a single section. + typedef uint16_t SectionIndex; // used for indexing data inside single sections. + typedef float MappingItem; // type in which SID and other mapping types are represented in. + typedef int32_t ExtraMappingItem; // type in which extra mapping items are represented in. + typedef float DataItem; // type in which the actual voltage, current, conductivity etc values + // are represented in + + enum SortingKey { SID, LOC }; // sorting specifier for sections. + // SID will make sections to be sorted by their mapping, + // LOC by their locaiton in file. + + namespace baseTypes { + // these typedefs define from which STL some of the clases are derived from + + typedef std::set FrameInfo; + typedef std::vector CompartmentMapping; + typedef std::vector SectionData; + // typedef std::map + // CellData; done blow + typedef std::map FrameOrganiser; + typedef std::vector ReadOptimiser; + } + + // end of type conventions + + class Header { + private: + bool isNative; + + // header info + uint32_t headerSize; + std::string libraryVersion; + std::string simulatorVersion; + CellCount amountOfCells; + FrameDataCount totalAmountOfCompartments; + FrameCount numberOfSteps; + Time startTime; + Time endTime; + Time dt; + std::string dataUnit; + std::string timeUnit; + FrameDataCount mappingSize; + std::string mappingName; + FrameDataCount extraMappingSize; + std::string extraMappingName; + std::string targetName; + + public: + Header() : headerSize(0){} + Header(char* readBuffer); + // Header(const Header& source); + + bool operator==(const Header& h1) const; + + // get methods: + inline bool sourceIsNative() const { + return isNative; + } + inline std::string getLibraryVersion() const { + return libraryVersion; + } + inline std::string getSimulatorVersion() const { + return simulatorVersion; + } + inline CellCount getNumberOfCells() const { + return amountOfCells; + } + inline FrameDataCount getTotalNumberOfCompartments() const { + return totalAmountOfCompartments; + } + inline FrameCount getNumberOfSteps() const { + return numberOfSteps; + } + inline std::string getMappingName() const { + return mappingName; + } + inline std::string getExtraMappingName() const { + return extraMappingName; + } + inline std::string getDataUnit() const { + return dataUnit; + } + inline std::string getTimeUnit() const { + return timeUnit; + } + inline FrameDataCount getMappingSize() const { + return mappingSize; + } + inline FrameDataCount getExtraMappingSize() const { + return extraMappingSize; + } + inline std::string getTargetName() const { + return targetName; + } + inline Time getStartTime() const { + return startTime; + } + inline Time getEndTime() const { + return endTime; + } + inline Time getTimeStepSize() const { + return dt; + } + // end get methods + }; + + class CellInfo { + private: + CellID cellNum; + CellDataCount amountOfCompartments; + + std::ios::off_type dataLocation; + std::ios::off_type extraMappingLocation; + std::ios::off_type mappingLocation; + + void* master; + + public: + enum key { GID, LOCATION }; + + CellInfo(); + CellInfo(char* readBuffer, bool isNative = true, key cmp = GID); + + bool operator==(const CellInfo& c1) const; + bool operator!=(const CellInfo& c1) const; + bool operator<=(const CellInfo& c1) const; + bool operator>=(const CellInfo& c1) const; + bool operator<(const CellInfo& c1) const; + bool operator>(const CellInfo& c1) const; + + inline std::ios::off_type getDataLocation() const { + return dataLocation; + } + + inline std::ios::off_type getMappingLocation() const { + return mappingLocation; + } + + inline std::ios::off_type getExtraMappingLocation() const { + return extraMappingLocation; + } + + inline CellID getCellNum() const { + return cellNum; + } + + inline CellDataCount getAmountOfCompartments() const { + return amountOfCompartments; + } + + static bool compareByOffset(const CellInfo& c1, const CellInfo& c2); + }; + + class FrameInfo : private baseTypes::FrameInfo { + private: + typedef baseTypes::FrameInfo Base; + typedef std::map Lookup; + Lookup gidLookup; + + public: + typedef Base::const_iterator const_iterator; + + FrameInfo() { + } + FrameInfo(char* buffer, + char* endOfBuffer, + bool isNative = true, + CellInfo::key sortMode = CellInfo::GID); + FrameInfo& filterGIDs(const std::vector& gids); + + /** + * Copy constructor. Needed to ensure set iterators reference the correct collection + */ + FrameInfo(const FrameInfo& rhs); + + /** + * Assignment operator. Needed to ensure set iterators reference the correct collection + */ + FrameInfo& operator=(const FrameInfo& rhs); + + inline const CellInfo& operator[](CellID gid) { + return *gidLookup[gid]; + } + + inline const_iterator begin() const { + return Base::begin(); + } + + inline const_iterator end() const { + return Base::end(); + } + }; + + class CompartmentMapping : public baseTypes::CompartmentMapping { + private: + typedef baseTypes::CompartmentMapping Base; + + public: + typedef Base::size_type SizeType; + + CompartmentMapping(); + CompartmentMapping(MappingItem* start, MappingItem* end); + CompartmentMapping(MappingItem* buffer, SizeType bufSize); + }; + + class SectionData : public baseTypes::SectionData { + private: + typedef baseTypes::SectionData Base; + + public: + class Identifier; + SectionData(); + SectionData(DataItem firstData); + SectionData(DataItem* start, DataItem* end); + + bool operator==(const SectionData& s1); + bool operator!=(const SectionData& s1); + + inline SectionDataCount numOfCompartments() { + return size(); + } + + static bool compareByOriginalLocation(const Identifier& c1, const Identifier& c2); + }; + + class SectionData::Identifier { + public: + private: + typedef SectionData::Identifier thisType; + + MappingItem sid; + SectionIndex loc; + + SortingKey key; + + public: + inline Identifier(const MappingItem& map, + const SectionIndex& location, + SortingKey primary = SID) { + sid = map, loc = location, key = primary; + } + + inline bool operator!=(const thisType& c) const { + return (key == SID) ? sid != c.sid : loc != c.loc; + } + + inline bool operator==(const thisType& c) const { + return (key == SID) ? sid == c.sid : loc == c.loc; + } + + inline bool operator<(const thisType& c) const { + return (key == SID) ? sid < c.sid : loc < c.loc; + } + + inline bool operator<=(const thisType& c) const { + return (key == SID) ? sid <= c.sid : loc <= c.loc; + } + + inline bool operator>(const thisType& c) const { + return (key == SID) ? sid > c.sid : loc > c.loc; + } + + inline bool operator>=(const thisType& c) const { + return (key == SID) ? sid >= c.sid : loc >= c.loc; + } + + inline operator MappingItem() const { + return sid; + } + + inline MappingItem getSID() const { + return sid; + } + + inline SectionIndex getLocation() const { + return loc; + } + }; + + namespace baseTypes { + typedef std::map CellData; + } + + class CellData : public baseTypes::CellData { + private: + typedef baseTypes::CellData Base; + typedef std::map Lookup; + + SortingKey sectSort; + Lookup sidLookup; + + public: + CellData(); + CellData(const SortingKey segmentSorting = SID); + CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping); + void fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping); + + inline iterator find(const SectionData::Identifier& i) { + return Base::find(i); + } + + inline iterator find(const MappingItem& map_find) { + Lookup::iterator match = sidLookup.find(map_find); + if (match == sidLookup.end()) + return end(); + else + return match->second; + } + + bool operator==(const CellData& d1); + }; + + class ReadManager : public std::ifstream { + private: + typedef std::ifstream Base; + + public: + typedef Base::off_type FileOffset; + typedef Base::pos_type FilePosition; + + private: + FileOffset cellInfoOrigin; // position in file where cell info starts (after header) + + bool isNative; // indicates whether file is originated from an architecture that arranges + // variables in same byte order + + FrameDataCount mappingSize_floats; + + FileOffset startLocation; + FileOffset mappingLocation; + + inline void seekgStart() { + seekg(startLocation, beg); + } + + static CellInfo makeNullInfo(); + + protected: + inline FileOffset getStartOffset() { + return startLocation; + } + + public: + static const CellInfo _null_info; + + Header fileInfo; + + ReadManager(); + + /** + * Constructor which immediately opens a file for reading. + * + * @param filePath The bbp binary report file to open + */ + explicit ReadManager(const std::string& filePath); + + /** + * Opens the filename specified provided this object has not already opened a file + * + * @param filePath The bbp binary report file to open + */ + void open(const std::string& filePath); + + // Find the cell's info which corresponds to cellNum (note: cellNum is not the cell index in + // the file) + CellInfo retrieveFindCell(CellID cellNum); + + // Retrieve global (time invariant) information of a cell (not actual data of the cell) + CellInfo retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode = CellInfo::LOCATION); + + // NOTE: its highly recommended to either disable cache (set to 0) or set cache size to + // unlimited (set to -1) before using this method: + FrameInfo retrieveAllCellInfo(CellInfo::key sortMode = CellInfo::LOCATION); + + /*seekg*: + * all the seekg functions below are wrapping around the original iostream seekg method. + */ + + inline void seekgCellInfo(CellIndex cellIndex = 0) { + seekg(cellInfoOrigin); + if (cellIndex) + seekg(cellIndex * SIZE_CELL_INFO_LENGTH, cur); + } + + inline void seekgData(const CellInfo& cellSpec, FrameIndex timeStep = 0) { + seekg(cellSpec.getDataLocation() + timeStep * getFrameSize_bytes(), beg); + } + + inline void seekgMapping(const CellInfo& cellSpec) { + seekg(cellSpec.getMappingLocation(), beg); + } + + inline void seekgExtraMapping(const CellInfo& cellSpec) { + seekg(cellSpec.getExtraMappingLocation(), beg); + } + + inline void seekgMappingStart() { + seekg(mappingLocation, beg); + } + + inline void seekgFrame(FrameIndex timeStep) { + seekg(startLocation + getFrameSize_bytes() * timeStep); + } + + /*read(): + * this function essentially takes 3 arguments (the third being the datatype). + * it does the same as the read function of the base ifstream class that it is overloading + * the only difference is that num is not neccesarily in number of bytes + * instead it defines the mount of elemetns in the buffer (depending on the datatype of the + * buffer). + * + * Important: + * This read method will automatically fix the byte orders of the data if applicable. + * The algorithm of fixing the byte orders depends on the size of each element in the buffer + * array. + * Therefore you should be carefull with passing char* buffers, + * unless you are really intending to read single byte data. + */ + + template + ReadManager& read(datatype* buffer, std::streamsize num) { + Base::read((char*)buffer, num * sizeof(datatype)); + + if (sizeof(datatype) != 1 && !isNative) // a good compiler should not include this + // entire if statement when this function is + // reading single byte items. + // This if is to avoid entering an empty loop. Otherwise could have been done in + // switch below. + for (datatype* endOfBuffer = buffer + num; buffer < endOfBuffer; buffer++) + switch (sizeof(datatype)) { // metamorphic switch + case 8: // double, int64_t, offsets etc + DOUBLESWAP(*buffer); + break; + case 4: // float, int32_t, etc + FIX_INT(*buffer); + break; + case 2: // short + FIX_SHORT(*buffer); + break; + default: // anything weird that cannot be fixed. + // TODO: throw error exception + break; + } + return *this; + } + + /*checkState(): + * will determine whether a reading error has occured, + * output the according error message if there has been an error, + * and exit execution if an error occured. + */ + + void checkState(); + + /*getFrameSize: + * returns the number of bytes that hold the data of all compartments of all cells for a + * single timestep. + * please note that its is in bytes, not floats! + * use fileInfo.getTotalNumberOfCompartments() instead for number of floats. + */ + + inline std::streamsize getFrameSize_bytes() const { + return fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem); + } + + /*getMappingSize: + * returns the number of bytes that hold the data of the mappings of all compartmetns of all + * cells. + * please note that this is in bytes not floats. + */ + + inline std::streamsize getMappingSize_bytes() const { + return mappingSize_floats * sizeof(MappingItem); + } + + // The following are low level c-style reading interfaces for max performance - they provide + // unsorted, raw data. + /*readFrameMapping: + * will fill buffer with mappings of all compartmetns of all cells. + * the file pointer will be automatically adjusted accordingly and then restored to original + * position. + * use getMappingSize() / sizeof(float) to determine the size of the buffer in number bytes + * (NOT float). + */ + + void readFrameMapping(MappingItem* buffer); + + /*readFrame: + * will fill dataBuffer with data from next frame. + * use seekgFrame() to specify timestep. + * use fileInfo.getTotalNumberOfCompartments() to determine the size of buffer + */ + void readFrame(DataItem* dataBuffer); + + private: + /** + * Upon opening a report file, read the header with metadata + */ + void readHeader(); + }; + + class FrameParser : protected ReadManager { + public: + typedef std::vector > CompartmentIndexing; + typedef std::vector > CompartmentCounts; + + protected: + class FrameOrganiser : public baseTypes::FrameOrganiser { + private: + SortingKey sectSort; + + public: + FrameOrganiser(); + explicit FrameOrganiser(SortingKey sectionSorting); + + void fillData(const FrameInfo& info, + const std::vector& mapping, + const float* dataBuffer); + + void makeOrderedDataBuffer(DataItem* buffer) const; + + CompartmentIndexing makeOrderedOffsetReferences() const; + CompartmentCounts makeOrderedSegSizes() const; + }; + + class ReadOptimiser : public baseTypes::ReadOptimiser { + private: + void updateOffsets(size_t newSize); + + protected: + FileOffset firstOffset; + FrameDataCount dataSize_elements; + + size_t elementSize; + static size_t defaultElementSize; + + public: + ReadOptimiser(); + explicit ReadOptimiser(std::streamsize frameSize_bytes); + ReadOptimiser(const FrameInfo& cellsToRead, + std::streamsize frameSize_bytes, + FilePosition startLocation); + + inline FileOffset getFirst() const { + return firstOffset; + } + + inline FrameDataCount getDataSize_elements() const { + return dataSize_elements; + } + + inline void setElementSize(size_t newSize) { + if (newSize != elementSize) { + updateOffsets(newSize); + elementSize = newSize; + } + } + + static inline void setDefaultElementSize(size_t newDefault) { + defaultElementSize = newDefault; + } + + inline size_t getElementSize() const { + return elementSize; + } + }; + + void readFrame(DataItem* buffer); + + private: + DataItem* readBuffer; + DataItem** refArray; + + ReadOptimiser offsets; + FrameOrganiser org; + + FrameIndex timestep; + + void createReferences(const FrameInfo& cinfo, bool sortData); + + void bufferTransfer(DataItem* target) const; + + public: + /*Constructor: + * Will read next frame of data and parse it into a map of CellData structure from file. + */ + FrameParser(); + FrameParser(const std::string& file, bool sortData = true); + FrameParser(const std::string& file, + const std::vector& gidTarget, + bool sortData = true); + ~FrameParser(); + + /*retarget: + * Takes a list of gids in form of a vector of CellID and uses it to respecify the target + * cells of this + * frameparser. + */ + + inline FrameIndex simtime2index(Time time) { + return (FrameIndex)((time - fileInfo.getStartTime()) / fileInfo.getTimeStepSize()); + } + + void retarget(const std::vector& gidTarget, bool sortData = true); + + inline const Header& getHeader() const { + return fileInfo; + } + + inline FrameDataCount getBufferSize_elements() const { + return offsets.getDataSize_elements(); + } + + // to select frame (similar use to iterators): + // Note that readFrameData already incorporates operator++ ! + FrameParser& operator++(); // increment timestep + FrameParser& operator++(int); // increment timestep + FrameParser& operator--(); // decrement timestep + FrameParser& operator--(int); // decrement timestep + FrameParser& operator+=(FrameDiff increment); // increment timestep by scalar + FrameParser& operator-=(FrameDiff decrement); // decrement timestep by scalar + FrameParser& operator=(FrameIndex newTime); // set timestep to scalar + + // returns the current timestep: + inline FrameIndex getTimestep() const { + return timestep; + } + + inline bool hasMore() const { + return timestep < fileInfo.getNumberOfSteps(); + } + + // returns the current time that the current timestep represents. + Time getTime() const; + + void readFrameMapping(MappingItem* buffer); + void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame + // after. Data is sorted by GID and segment num + + /*getReferences() + * returns a vector of vectors of offsets that correspond to the relevant segments inside + * the read buffer. + * eg. + * getReferences()[12][31]; //will give the offset for segment 31 in cell 12 + * if the segment has no comparments, it will return an offset of 0. + */ + inline CompartmentIndexing getReferences() const { + return org.makeOrderedOffsetReferences(); + } + + /*getCompartmentCounts() + * returns a vector of vectors like getReferences() above, + * but instead of containing the offsets for each segment, + * it contains the amount of compartmetns of the corresponding segment. + */ + inline CompartmentCounts getCompartmentCounts() const { + return org.makeOrderedSegSizes(); + } + }; +} +#endif diff --git a/converter/binary_reader/crossPlatformConversion.h b/converter/binary_reader/crossPlatformConversion.h new file mode 100644 index 0000000..7e33343 --- /dev/null +++ b/converter/binary_reader/crossPlatformConversion.h @@ -0,0 +1,28 @@ +// Author: Dr Konstantinos Sfyrakis + +#ifndef CROSSPLATFORMCONVERSION_H_ +#define CROSSPLATFORMCONVERSION_H_ + +#define SMALL_NUM 0.00000001 // anything that avoids division overflow +#define ABS(x) ((x) >= 0 ? (x) : -(x)) // absolute value + +#define SWAP_2(x) ((((x)&0xff) << 8) | ((unsigned short)(x) >> 8)) +#define SWAP_4(x) \ + (((x) << 24) | (((x) << 8) & 0x00ff0000) | (((x) >> 8) & 0x0000ff00) | ((x) >> 24)) +#define FIX_SHORT(x) (*(unsigned short*)&(x) = SWAP_2(*(unsigned short*)&(x))) +#define FIX_INT(x) (*(unsigned int*)&(x) = SWAP_4(*(unsigned int*)&(x))) +#define FIX_LONG(x) FIX_INT(x) +#define FIX_FLOAT(x) FIX_INT(x) +#define DOUBLESWAP(x) ByteSwap((unsigned char*)&x, sizeof(x)) + +inline void ByteSwap(unsigned char* b, int n) { + int i = 0; + int j = n - 1; + + while (i < j) { + std::swap(b[i], b[j]); + i++, j--; + } +} + +#endif /*CROSSPLATFORMCONVERSION_H_*/ diff --git a/converter/reports_converter.cpp b/converter/reports_converter.cpp new file mode 100644 index 0000000..f9295f5 --- /dev/null +++ b/converter/reports_converter.cpp @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include + +#include +#include +#include "binary_reader/ReadManager.h" + +using namespace bbpReader; + +int main(int argc, char *argv[]) { + + if(argc!=2) { + logger->error("Wrong number of arguments."); + logger->info("Try: ./reports_converter example.bbp"); + return 0; + } + const char* file_name = argv[1]; + logger->info("Trying to convert '{}' binary report...'", file_name); + std::ifstream f(file_name); + if(!f.good()) { + logger->error("File '{}' not found!", file_name); + return 0; + } + + FrameParser frame_parser(file_name); + Header header = frame_parser.getHeader(); + + std::string filename = file_name; + std::string report_name = filename.substr(0, filename.find(".")); + + // Get header information in order to create the report + double tstart = header.getStartTime(); + double tstop = header.getEndTime(); + double dt = header.getTimeStepSize(); + sonata_create_report(report_name.data(), tstart, tstop, dt, "mV", "compartment"); + + // Get Cell information to create node/element structure + // TODO: check if it could be done without getting the FrameParser per gid + ReadManager file(file_name); + FrameInfo cells = file.retrieveAllCellInfo(); + std::vector node_ids(1); + for (FrameInfo::const_iterator it = cells.begin(); it != cells.end(); ++it) { + node_ids[0] = it->getCellNum(); + sonata_add_node(report_name.data(), "All", 0, it->getCellNum()); + + FrameParser frame_parser_gid(file_name, node_ids); + int num_element_ids = frame_parser_gid.getBufferSize_elements(); + DataItem* element_ids_buffer = new DataItem[num_element_ids]; + frame_parser_gid.readFrameMapping(element_ids_buffer); + + std::vector element_ids(element_ids_buffer, element_ids_buffer + num_element_ids); + for (auto& element : element_ids) { + sonata_add_element(report_name.data(), "All", node_ids[0], element, nullptr); + } + } + // Generate the initial structure + sonata_prepare_datasets(); + + int total_num_element_ids = frame_parser.getBufferSize_elements(); + DataItem* all_element_ids_buffer = new DataItem[total_num_element_ids]; + // Write the timestep frames + while (frame_parser.hasMore()) { + frame_parser.readFrameData(all_element_ids_buffer); + std::vector data_element_ids(all_element_ids_buffer, all_element_ids_buffer + total_num_element_ids); + sonata_write_buffered_data(report_name.data(), data_element_ids.data(), data_element_ids.size(), 1); + } + sonata_clear(); + logger->info("File '{}' successfully converted to '{}.h5'", file_name, report_name); + return 0; +} \ No newline at end of file diff --git a/converter/spikes_converter.cpp b/converter/spikes_converter.cpp new file mode 100644 index 0000000..a6ebe2b --- /dev/null +++ b/converter/spikes_converter.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include +#include + +int main() { + + std::ifstream infile("out.dat"); + if (!infile) { + logger->error("File out.dat not found!"); + return 0; + } + + // remove /scatter + std::string scatter; + getline(infile, scatter); + if (scatter != "/scatter") { + infile.seekg(0, std::ios::beg); + } + + std::vector spike_timestamps; + std::vector spike_node_ids; + double timestamp; + int node_id; + + while (infile >> timestamp >> node_id) { + spike_timestamps.push_back(timestamp); + spike_node_ids.push_back(node_id); + } + + // Create a spike file + sonata_create_spikefile(".", "out"); + + std::string population_name = "All"; + uint64_t population_offset = 0; + sonata_add_spikes_population(population_name.data(), + population_offset, + spike_timestamps.data(), + spike_timestamps.size(), + spike_node_ids.data(), + spike_node_ids.size()); + sonata_write_spike_populations(); + // Close the spike file + sonata_close_spikefile(); + + logger->info("File 'out.dat' successfully converted to 'out.h5'"); + + return 0; +} \ No newline at end of file diff --git a/include/bbp/sonata/reports.h b/include/bbp/sonata/reports.h index 651ef01..7d0a9f8 100644 --- a/include/bbp/sonata/reports.h +++ b/include/bbp/sonata/reports.h @@ -119,6 +119,15 @@ int sonata_record_node_data(double step, */ int sonata_record_data(double step); +/** + * \brief Write steps_to_write steps given a pre-buffered data + * \return -3 if the Sonata report doesn't exist, -1 if the report name doesn't exist, 0 otherwise + */ +int sonata_write_buffered_data(const char* report_name, + const float* buffered_data, + size_t buffered_data_size, + uint32_t steps_to_write); + /** * \brief Check status of the recordings/buffers and flush if necessary * \return 0 diff --git a/src/data/sonata_data.cpp b/src/data/sonata_data.cpp index 169e221..2bc05d9 100644 --- a/src/data/sonata_data.cpp +++ b/src/data/sonata_data.cpp @@ -163,7 +163,7 @@ void SonataData::record_data(double step) { last_step_recorded_ += reporting_period_; if (current_step_ == steps_to_write_) { - write_data(); + flush(); } } @@ -199,7 +199,7 @@ void SonataData::check_and_write(double timestep) { remaining_steps_, steps_recorded_); } - write_data(); + flush(); } steps_recorded_ = 0; } @@ -313,26 +313,30 @@ void SonataData::write_spike_populations() { } } -void SonataData::write_data() { +void SonataData::write_data(const std::vector& buffered_data, uint32_t steps_to_write) { if (remaining_steps_ <= 0) { // Nothing left to write return; } - if (current_step_ >= remaining_steps_) { // Avoid writing out of bounds - current_step_ = remaining_steps_; + if (steps_to_write >= remaining_steps_) { // Avoid writing out of bounds + steps_to_write = remaining_steps_; } if (SonataReport::rank_ == 0) { logger->debug("WRITING timestep data to file {} in population {}", report_name_, population_name_); } - hdf5_writer_->write_2D(report_buffer_, current_step_, total_elements_); - remaining_steps_ -= current_step_; + hdf5_writer_->write_2D(buffered_data, steps_to_write , total_elements_); + remaining_steps_ -= steps_to_write ; if (SonataReport::rank_ == 0) { - logger->debug("\t-Steps written: {}", current_step_); + logger->debug("\t-Steps written: {}", steps_to_write ); logger->debug("\t-Remaining steps: {}", remaining_steps_); } last_position_ = 0; - current_step_ = 0; + steps_to_write = 0; +} + +void SonataData::flush() { + write_data(report_buffer_, current_step_); } void SonataData::close() { diff --git a/src/data/sonata_data.h b/src/data/sonata_data.h index ad5397e..4b86f9f 100644 --- a/src/data/sonata_data.h +++ b/src/data/sonata_data.h @@ -73,7 +73,8 @@ class SonataData void write_spike_populations(); void add_population(std::unique_ptr&& population); - void write_data(); + void write_data(const std::vector& buffered_data, uint32_t steps_to_write); + void flush(); void close(); bool is_due_to_report(double step) const noexcept; diff --git a/src/library/report.cpp b/src/library/report.cpp index 6b4393e..cef9327 100644 --- a/src/library/report.cpp +++ b/src/library/report.cpp @@ -113,6 +113,12 @@ void Report::record_data(double step) { } } +void Report::write_buffered_data(const std::vector& buffered_data, uint32_t steps_to_write) { + for (const auto& sonata_data : sonata_populations_) { + sonata_data->write_data(buffered_data, steps_to_write); + } +} + void Report::check_and_flush(double timestep) { for (const auto& sonata_data : sonata_populations_) { sonata_data->check_and_write(timestep); @@ -134,7 +140,7 @@ void Report::flush(double time) { } for (const auto& sonata_data : sonata_populations_) { // Write if there are any remaining steps to write - sonata_data->write_data(); + sonata_data->flush(); if (time - tend_ + dt_ / 2 > 1e-6) { sonata_data->close(); } diff --git a/src/library/report.h b/src/library/report.h index eeb3c26..02db309 100644 --- a/src/library/report.h +++ b/src/library/report.h @@ -45,6 +45,8 @@ class Report virtual void record_data(double step, const std::vector& node_ids); virtual void record_data(double step); + virtual void write_buffered_data(const std::vector& buffered_data, + uint32_t steps_to_write); virtual void check_and_flush(double timestep); virtual void flush(double time); void refresh_pointers(std::function refresh_function); diff --git a/src/library/reports.cpp b/src/library/reports.cpp index e23d858..189bc97 100644 --- a/src/library/reports.cpp +++ b/src/library/reports.cpp @@ -104,6 +104,22 @@ int sonata_record_data(double step) { return 0; } +int sonata_write_buffered_data(const char* report_name, + const float* buffered_data, + size_t buffered_data_size, + uint32_t steps_to_write) { + if (sonata_report.is_empty()) { + return -3; + } + if (!sonata_report.report_exists(report_name)) { + return -1; + } + const std::vector data(buffered_data, buffered_data + buffered_data_size); + auto report = sonata_report.get_report(report_name); + report->write_buffered_data(data, steps_to_write); + return 0; +} + int sonata_check_and_flush(double timestep) { auto functor = std::mem_fn(&bbp::sonata::Report::check_and_flush); sonata_report.apply_all(functor, timestep); diff --git a/tests/integration/integration_test.cpp b/tests/integration/integration_test.cpp index a098201..a3a797e 100644 --- a/tests/integration/integration_test.cpp +++ b/tests/integration/integration_test.cpp @@ -113,10 +113,11 @@ void init(const char* report_name, } } -void change_data(std::vector& neurons) { +void change_data(std::vector& neurons, std::vector& buffered_data) { // Increment in 1 per timestep every voltage for (auto& neuron : neurons) { for (auto& element : neuron.voltages) { + buffered_data.push_back(element); element++; } } @@ -176,6 +177,9 @@ int main() { const char* soma_report = "soma_report"; const char* single_report = "single_report"; const char* units = "mV"; + std::vector element_buffered_data = {}; + std::vector soma_buffered_data = {}; + std::vector single_neurons_buffered_data = {}; init(element_report, tstart, tstop, dt, element_neurons, "compartment", units); init(soma_report, tstart, tstop, dt, soma_neurons, "soma", units); @@ -211,11 +215,32 @@ int main() { sonata_check_and_flush(t); t += dt; // Change data every timestep - change_data(element_neurons); - change_data(soma_neurons); - change_data(single_neurons); + change_data(element_neurons, element_buffered_data); + change_data(soma_neurons, soma_buffered_data); + change_data(single_neurons, single_neurons_buffered_data); } sonata_flush(t); + sonata_clear(); + + // Write pre-buffered data (GPU usecase) + if (global_rank == 0) { + logger->info("GPU usecase: Writting prebuffered steps"); + } + const char* buffered_soma_report = "buffered_soma_report"; + + init(buffered_soma_report, + tstart, + tstop, + dt, + soma_neurons, + "soma", + units); + + sonata_setup_communicators(); + sonata_prepare_datasets(); + sonata_write_buffered_data(buffered_soma_report, soma_buffered_data.data(), soma_buffered_data.size(), num_steps); + sonata_clear(); + const std::string output_dir = "."; // Create a spike file From d9084c00b2fa64464d1a78157d690dc67f9ea928 Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Thu, 21 Jul 2022 18:31:58 +0200 Subject: [PATCH 02/10] Separate between soma and compartment converter --- CMakeLists.txt | 2 +- {converter => tools/converter}/CMakeLists.txt | 5 +- .../binary_reader/BinaryPositions.h | 0 .../converter}/binary_reader/CMakeLists.txt | 0 .../converter}/binary_reader/ReadManager.cpp | 5 ++ .../converter}/binary_reader/ReadManager.h | 1 + .../binary_reader/crossPlatformConversion.h | 0 .../converter}/reports_converter.cpp | 63 ++++++++++++------- .../converter}/spikes_converter.cpp | 29 ++++++--- 9 files changed, 71 insertions(+), 34 deletions(-) rename {converter => tools/converter}/CMakeLists.txt (91%) rename {converter => tools/converter}/binary_reader/BinaryPositions.h (100%) rename {converter => tools/converter}/binary_reader/CMakeLists.txt (100%) rename {converter => tools/converter}/binary_reader/ReadManager.cpp (99%) rename {converter => tools/converter}/binary_reader/ReadManager.h (99%) rename {converter => tools/converter}/binary_reader/crossPlatformConversion.h (100%) rename {converter => tools/converter}/reports_converter.cpp (51%) rename {converter => tools/converter}/spikes_converter.cpp (57%) diff --git a/CMakeLists.txt b/CMakeLists.txt index b249365..68ef35b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,5 +164,5 @@ endif() # Converter of binary to SONATA reports # ============================================================================= if(SONATA_REPORT_ENABLE_CONVERTER) - add_subdirectory(converter) + add_subdirectory(tools/converter) endif() diff --git a/converter/CMakeLists.txt b/tools/converter/CMakeLists.txt similarity index 91% rename from converter/CMakeLists.txt rename to tools/converter/CMakeLists.txt index 1b2a771..e209492 100644 --- a/converter/CMakeLists.txt +++ b/tools/converter/CMakeLists.txt @@ -33,4 +33,7 @@ target_link_libraries(reports_converter PUBLIC binary_reader PRIVATE spdlog::spdlog_header_only PRIVATE ${MPI_CXX_LIBRARIES} -) \ No newline at end of file +) + +install(TARGETS spikes_converter DESTINATION bin) +install(TARGETS reports_converter DESTINATION bin) \ No newline at end of file diff --git a/converter/binary_reader/BinaryPositions.h b/tools/converter/binary_reader/BinaryPositions.h similarity index 100% rename from converter/binary_reader/BinaryPositions.h rename to tools/converter/binary_reader/BinaryPositions.h diff --git a/converter/binary_reader/CMakeLists.txt b/tools/converter/binary_reader/CMakeLists.txt similarity index 100% rename from converter/binary_reader/CMakeLists.txt rename to tools/converter/binary_reader/CMakeLists.txt diff --git a/converter/binary_reader/ReadManager.cpp b/tools/converter/binary_reader/ReadManager.cpp similarity index 99% rename from converter/binary_reader/ReadManager.cpp rename to tools/converter/binary_reader/ReadManager.cpp index d44cbb5..eae6c3f 100644 --- a/converter/binary_reader/ReadManager.cpp +++ b/tools/converter/binary_reader/ReadManager.cpp @@ -767,6 +767,11 @@ namespace bbpReader { // read. } + void FrameParser::readMultipleFrameData(DataItem* buffer, int num_frames) { + readFrame(buffer); + timestep += num_frames; + } + size_t FrameParser::ReadOptimiser::defaultElementSize = 1; FrameParser::ReadOptimiser::ReadOptimiser() { diff --git a/converter/binary_reader/ReadManager.h b/tools/converter/binary_reader/ReadManager.h similarity index 99% rename from converter/binary_reader/ReadManager.h rename to tools/converter/binary_reader/ReadManager.h index 97abe37..7b054e3 100644 --- a/converter/binary_reader/ReadManager.h +++ b/tools/converter/binary_reader/ReadManager.h @@ -697,6 +697,7 @@ namespace bbpReader { void readFrameMapping(MappingItem* buffer); void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame // after. Data is sorted by GID and segment num + void readMultipleFrameData(DataItem* buffer, int num_frames); /*getReferences() * returns a vector of vectors of offsets that correspond to the relevant segments inside diff --git a/converter/binary_reader/crossPlatformConversion.h b/tools/converter/binary_reader/crossPlatformConversion.h similarity index 100% rename from converter/binary_reader/crossPlatformConversion.h rename to tools/converter/binary_reader/crossPlatformConversion.h diff --git a/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp similarity index 51% rename from converter/reports_converter.cpp rename to tools/converter/reports_converter.cpp index f9295f5..702bd64 100644 --- a/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -1,26 +1,27 @@ -#include -#include -#include #include +#include +#include #include +#include +#include "binary_reader/ReadManager.h" #include #include -#include "binary_reader/ReadManager.h" using namespace bbpReader; -int main(int argc, char *argv[]) { - - if(argc!=2) { +int main(int argc, char* argv[]) { + if (argc != 3) { logger->error("Wrong number of arguments."); - logger->info("Try: ./reports_converter example.bbp"); + logger->info("Try: ./reports_converter <[--soma, --compartment]>"); + logger->info("Example: ./reports_converter soma.bbp --soma"); return 0; } const char* file_name = argv[1]; + std::string report_type = argv[2]; logger->info("Trying to convert '{}' binary report...'", file_name); std::ifstream f(file_name); - if(!f.good()) { + if (!f.good()) { logger->error("File '{}' not found!", file_name); return 0; } @@ -29,12 +30,13 @@ int main(int argc, char *argv[]) { Header header = frame_parser.getHeader(); std::string filename = file_name; - std::string report_name = filename.substr(0, filename.find(".")); + std::string report_name = filename.substr(filename.find_last_of("/\\") + 1); // Get header information in order to create the report double tstart = header.getStartTime(); double tstop = header.getEndTime(); double dt = header.getTimeStepSize(); + logger->info("Report info: tstart = '{}', tstop = '{}', dt = '{}'", tstart, tstop, dt); sonata_create_report(report_name.data(), tstart, tstop, dt, "mV", "compartment"); // Get Cell information to create node/element structure @@ -46,28 +48,45 @@ int main(int argc, char *argv[]) { node_ids[0] = it->getCellNum(); sonata_add_node(report_name.data(), "All", 0, it->getCellNum()); - FrameParser frame_parser_gid(file_name, node_ids); - int num_element_ids = frame_parser_gid.getBufferSize_elements(); - DataItem* element_ids_buffer = new DataItem[num_element_ids]; - frame_parser_gid.readFrameMapping(element_ids_buffer); + if (report_type == "--soma") { + sonata_add_element(report_name.data(), "All", node_ids[0], 0, nullptr); + } else { // --compartment + FrameParser frame_parser_gid(file_name, node_ids); + int num_element_ids = frame_parser_gid.getBufferSize_elements(); + DataItem* element_ids_buffer = new DataItem[num_element_ids]; + frame_parser_gid.readFrameMapping(element_ids_buffer); - std::vector element_ids(element_ids_buffer, element_ids_buffer + num_element_ids); - for (auto& element : element_ids) { - sonata_add_element(report_name.data(), "All", node_ids[0], element, nullptr); + std::vector element_ids(element_ids_buffer, + element_ids_buffer + num_element_ids); + for (auto& element : element_ids) { + sonata_add_element(report_name.data(), "All", node_ids[0], element, nullptr); + } } } // Generate the initial structure sonata_prepare_datasets(); int total_num_element_ids = frame_parser.getBufferSize_elements(); - DataItem* all_element_ids_buffer = new DataItem[total_num_element_ids]; + int num_frames = 1; + DataItem* all_element_ids_buffer = new DataItem[total_num_element_ids * num_frames]; + int timestep = 0; // Write the timestep frames while (frame_parser.hasMore()) { - frame_parser.readFrameData(all_element_ids_buffer); - std::vector data_element_ids(all_element_ids_buffer, all_element_ids_buffer + total_num_element_ids); - sonata_write_buffered_data(report_name.data(), data_element_ids.data(), data_element_ids.size(), 1); + frame_parser.readMultipleFrameData(all_element_ids_buffer, num_frames); + std::vector data_element_ids(all_element_ids_buffer, + all_element_ids_buffer + total_num_element_ids); + + sonata_write_buffered_data(report_name.data(), + data_element_ids.data(), + data_element_ids.size(), + num_frames); + + if (timestep % 1000 == 0) { + logger->info("Writing timestep '{}'", timestep); + } + timestep += num_frames; } - sonata_clear(); + logger->info("'{}' timesteps written!", timestep); logger->info("File '{}' successfully converted to '{}.h5'", file_name, report_name); return 0; } \ No newline at end of file diff --git a/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp similarity index 57% rename from converter/spikes_converter.cpp rename to tools/converter/spikes_converter.cpp index a6ebe2b..de792ab 100644 --- a/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -1,16 +1,25 @@ -#include +#include #include +#include #include -#include #include #include -int main() { +int main(int argc, char* argv[]) { + const char* file_name = "out.dat"; + if (argc == 2) { + file_name = argv[1]; - std::ifstream infile("out.dat"); + } else if (argc > 2) { + logger->error("Wrong number of arguments."); + logger->info("Try: ./spikes_converter out.dat"); + return 0; + } + logger->info("Trying to convert '{}' binary report...'", file_name); + std::ifstream infile(file_name); if (!infile) { - logger->error("File out.dat not found!"); + logger->error("File {} not found!", file_name); return 0; } @@ -37,11 +46,11 @@ int main() { std::string population_name = "All"; uint64_t population_offset = 0; sonata_add_spikes_population(population_name.data(), - population_offset, - spike_timestamps.data(), - spike_timestamps.size(), - spike_node_ids.data(), - spike_node_ids.size()); + population_offset, + spike_timestamps.data(), + spike_timestamps.size(), + spike_node_ids.data(), + spike_node_ids.size()); sonata_write_spike_populations(); // Close the spike file sonata_close_spikefile(); From 7c821a8793ab47978bceadfb7e30f8aba44360ce Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Fri, 16 Sep 2022 18:12:07 +0200 Subject: [PATCH 03/10] Improve performance - Write several timesteps at a time (default = 10) - Avoid creation of innecesary std::vector --- tools/converter/reports_converter.cpp | 34 ++++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index 702bd64..d57d80d 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -10,6 +10,15 @@ using namespace bbpReader; +int get_steps_to_write (const char* var_name) { + int steps_to_write = 10; + char* var_value = getenv(var_name); + if (var_value != nullptr && atoi(var_value) > 0) { + steps_to_write = atoi(var_value); + } + return steps_to_write; +} + int main(int argc, char* argv[]) { if (argc != 3) { logger->error("Wrong number of arguments."); @@ -66,22 +75,25 @@ int main(int argc, char* argv[]) { // Generate the initial structure sonata_prepare_datasets(); - int total_num_element_ids = frame_parser.getBufferSize_elements(); - int num_frames = 1; - DataItem* all_element_ids_buffer = new DataItem[total_num_element_ids * num_frames]; - int timestep = 0; + uint32_t num_steps = header.getNumberOfSteps(); + uint32_t num_frames = get_steps_to_write("STEPS_TO_WRITE"); + uint64_t element_ids_per_frame = frame_parser.getBufferSize_elements(); + uint64_t total_element_ids = element_ids_per_frame * num_frames; + logger->info("Number of frames to write per iteration: '{}'", num_frames); + DataItem* all_element_ids_buffer = new DataItem[total_element_ids]; + uint32_t timestep = 0; // Write the timestep frames while (frame_parser.hasMore()) { + // Check if number of frames to write is not bigger than the remaining + if (num_steps - timestep < num_frames) { + num_frames = num_steps - timestep; + } frame_parser.readMultipleFrameData(all_element_ids_buffer, num_frames); - std::vector data_element_ids(all_element_ids_buffer, - all_element_ids_buffer + total_num_element_ids); - sonata_write_buffered_data(report_name.data(), - data_element_ids.data(), - data_element_ids.size(), + all_element_ids_buffer, + total_element_ids, num_frames); - - if (timestep % 1000 == 0) { + if (timestep % 1000 == 0 && timestep > 0) { logger->info("Writing timestep '{}'", timestep); } timestep += num_frames; From 3664c584c62286a4031edb117a0551453a54a82a Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Wed, 5 Oct 2022 17:03:17 +0200 Subject: [PATCH 04/10] Improve usage/help CLI --- tools/converter/reports_converter.cpp | 40 ++++++++++++++++++++------- tools/converter/spikes_converter.cpp | 37 +++++++++++++++++-------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index d57d80d..11da9c8 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -10,29 +10,49 @@ using namespace bbpReader; -int get_steps_to_write (const char* var_name) { +int get_steps_to_write(const char* var_name) { + // In general, writing 10 steps per iteration is best performance wise + // The environment variable STEPS_TO_WRITE could be overwritten for testing int steps_to_write = 10; char* var_value = getenv(var_name); if (var_value != nullptr && atoi(var_value) > 0) { - steps_to_write = atoi(var_value); + steps_to_write = atoi(var_value); } return steps_to_write; } +static void show_usage(std::string name) { + std::cerr << "Usage: " << name << " \n" + << "Options:\n" + << "\t-h,--help\t\t\tShow this help message\n" + << "\t<[--soma, --compartment]>\tSelect soma or compartment report\n" + << "Examples:\n" + << "\t " << name << " soma.bbp --soma\n" + << "\t " << name << " compartment.bbp --compartment" << std::endl; +} + +bool help(int argc, char* argv[]) { + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return true; + } + } + return false; +} + int main(int argc, char* argv[]) { - if (argc != 3) { - logger->error("Wrong number of arguments."); - logger->info("Try: ./reports_converter <[--soma, --compartment]>"); - logger->info("Example: ./reports_converter soma.bbp --soma"); - return 0; + if (argc != 3 || help(argc, argv)) { + show_usage(argv[0]); + return -1; } - const char* file_name = argv[1]; - std::string report_type = argv[2]; + const std::string file_name = argv[1]; + const std::string report_type = argv[2]; logger->info("Trying to convert '{}' binary report...'", file_name); std::ifstream f(file_name); if (!f.good()) { logger->error("File '{}' not found!", file_name); - return 0; + return -2; } FrameParser frame_parser(file_name); diff --git a/tools/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp index de792ab..820a246 100644 --- a/tools/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -6,21 +6,36 @@ #include #include +static void show_usage(std::string name) { + std::cerr << "Usage: " << name << " \n" + << "Options:\n" + << "\t-h,--help\tShow this help message\n" + << "Example:\n" + << "\t " << name << " out.dat" << std::endl; +} + +bool help(int argc, char* argv[]) { + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return true; + } + } + return false; +} + int main(int argc, char* argv[]) { - const char* file_name = "out.dat"; - if (argc == 2) { - file_name = argv[1]; - - } else if (argc > 2) { - logger->error("Wrong number of arguments."); - logger->info("Try: ./spikes_converter out.dat"); - return 0; + if (argc != 2 || help(argc, argv)) { + show_usage(argv[0]); + return -1; } + + const std::string file_name = argv[1]; logger->info("Trying to convert '{}' binary report...'", file_name); std::ifstream infile(file_name); if (!infile) { logger->error("File {} not found!", file_name); - return 0; + return -2; } // remove /scatter @@ -41,7 +56,7 @@ int main(int argc, char* argv[]) { } // Create a spike file - sonata_create_spikefile(".", "out"); + sonata_create_spikefile(".", file_name.data()); std::string population_name = "All"; uint64_t population_offset = 0; @@ -55,7 +70,7 @@ int main(int argc, char* argv[]) { // Close the spike file sonata_close_spikefile(); - logger->info("File 'out.dat' successfully converted to 'out.h5'"); + logger->info("File '{}' successfully converted to '{}.h5'", file_name, file_name); return 0; } \ No newline at end of file From 6e77797b271168d02a1628c31b41a887cbc48069 Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Thu, 6 Oct 2022 15:31:32 +0200 Subject: [PATCH 05/10] Using MPI calls in case of parallel implementation --- tools/converter/reports_converter.cpp | 11 +++++++++++ tools/converter/spikes_converter.cpp | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index 11da9c8..d10aff4 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -4,6 +4,10 @@ #include #include +#ifdef SONATA_REPORT_HAVE_MPI +#include +#endif + #include "binary_reader/ReadManager.h" #include #include @@ -42,6 +46,9 @@ bool help(int argc, char* argv[]) { } int main(int argc, char* argv[]) { +#ifdef SONATA_REPORT_HAVE_MPI + MPI_Init(nullptr, nullptr); +#endif if (argc != 3 || help(argc, argv)) { show_usage(argv[0]); return -1; @@ -120,5 +127,9 @@ int main(int argc, char* argv[]) { } logger->info("'{}' timesteps written!", timestep); logger->info("File '{}' successfully converted to '{}.h5'", file_name, report_name); + +#ifdef SONATA_REPORT_HAVE_MPI + MPI_Finalize(); +#endif return 0; } \ No newline at end of file diff --git a/tools/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp index 820a246..caf9981 100644 --- a/tools/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -3,6 +3,10 @@ #include #include +#ifdef SONATA_REPORT_HAVE_MPI +#include +#endif + #include #include @@ -25,6 +29,9 @@ bool help(int argc, char* argv[]) { } int main(int argc, char* argv[]) { +#ifdef SONATA_REPORT_HAVE_MPI + MPI_Init(nullptr, nullptr); +#endif if (argc != 2 || help(argc, argv)) { show_usage(argv[0]); return -1; @@ -72,5 +79,8 @@ int main(int argc, char* argv[]) { logger->info("File '{}' successfully converted to '{}.h5'", file_name, file_name); +#ifdef SONATA_REPORT_HAVE_MPI + MPI_Finalize(); +#endif return 0; } \ No newline at end of file From 3d002082d889d4803c93b47252f5a05f8c2dd72b Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Fri, 7 Oct 2022 11:52:42 +0200 Subject: [PATCH 06/10] Add converter test Write 1 frame per iteration --- .github/workflows/run_test.yaml | 4 +- CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 3 + tests/converter/CMakeLists.txt | 6 + tests/converter/converter_test.sh.in | 26 + tests/converter/soma.ref | Bin 0 -> 16768 bytes tests/converter/soma.ref.h5 | Bin 0 -> 23784 bytes tests/converter/spikes.ref | 1148 +++++++++++++++++ tests/converter/spikes.ref.h5 | Bin 0 -> 28072 bytes tests/integration/CMakeLists.txt | 2 +- tests/unit/CMakeLists.txt | 2 +- tools/converter/CMakeLists.txt | 4 +- tools/converter/binary_reader/ReadManager.cpp | 5 - tools/converter/binary_reader/ReadManager.h | 1 - tools/converter/reports_converter.cpp | 33 +- tools/converter/spikes_converter.cpp | 5 +- 17 files changed, 1201 insertions(+), 42 deletions(-) create mode 100644 tests/converter/CMakeLists.txt create mode 100644 tests/converter/converter_test.sh.in create mode 100644 tests/converter/soma.ref create mode 100644 tests/converter/soma.ref.h5 create mode 100644 tests/converter/spikes.ref create mode 100644 tests/converter/spikes.ref.h5 diff --git a/.github/workflows/run_test.yaml b/.github/workflows/run_test.yaml index 3f2a531..0fb6a4d 100644 --- a/.github/workflows/run_test.yaml +++ b/.github/workflows/run_test.yaml @@ -10,8 +10,8 @@ jobs: matrix: os: [ ubuntu-18.04 ] config: - - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON", mpi: ON} - - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=OFF", mpi: OFF} + - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_CONVERTER=ON", mpi: ON} + - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=OFF -DSONATA_REPORT_ENABLE_CONVERTER=ON", mpi: OFF} steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 68ef35b..1de885c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,7 +97,7 @@ if(SONATA_REPORT_ENABLE_MPI) else() if (HDF5_FOUND) # Avoid NOTFOUND error - set(MPI_CXX_LIBRARIES "") + set(MPI_C_LIBRARIES "") message(STATUS "HDF5 found, Using reporting serial implementation") else() message(FATAL_ERROR "No MPI or HDF5 found") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2acc247..d01f3fe 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -40,7 +40,7 @@ set_target_properties(sonata_report ) target_link_libraries(sonata_report PRIVATE spdlog::spdlog_header_only - PRIVATE ${MPI_CXX_LIBRARIES} + PRIVATE ${MPI_C_LIBRARIES} PRIVATE ${HDF5_C_LIBRARIES} ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e6f8e15..253d442 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,2 +1,5 @@ add_subdirectory(unit) add_subdirectory(integration) +if(SONATA_REPORT_ENABLE_CONVERTER) + add_subdirectory(converter) +endif() diff --git a/tests/converter/CMakeLists.txt b/tests/converter/CMakeLists.txt new file mode 100644 index 0000000..09b3b52 --- /dev/null +++ b/tests/converter/CMakeLists.txt @@ -0,0 +1,6 @@ +configure_file(converter_test.sh.in converter_test.sh @ONLY) + +add_test(NAME converter_test + COMMAND "/bin/sh" ${CMAKE_CURRENT_BINARY_DIR}/converter_test.sh + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" +) diff --git a/tests/converter/converter_test.sh.in b/tests/converter/converter_test.sh.in new file mode 100644 index 0000000..5b32366 --- /dev/null +++ b/tests/converter/converter_test.sh.in @@ -0,0 +1,26 @@ +#! /bin/sh + +@PROJECT_BINARY_DIR@/tools/converter/spikes_converter @CMAKE_CURRENT_SOURCE_DIR@/spikes.ref +@PROJECT_BINARY_DIR@/tools/converter/reports_converter @CMAKE_CURRENT_SOURCE_DIR@/soma.ref --soma +ref_spikes=@CMAKE_CURRENT_SOURCE_DIR@/spikes.ref.h5 +ref_soma=@CMAKE_CURRENT_SOURCE_DIR@/soma.ref.h5 + +h5diff -c spikes.ref.h5 $ref_spikes /spikes/All/timestamps > diff_spikes.dat 2>&1 +h5diff -c spikes.ref.h5 $ref_spikes /spikes/All/node_ids > diff_spikes.dat 2>&1 +h5diff -c soma.ref.h5 $ref_soma /report/All/data > diff_soma.dat 2>&1 + +if [ -s diff_spikes.dat ] +then + echo "Spike results are different, check the file diff_spikes.dat. Test failed!" + cat diff_spikes.dat + exit 1 +elif [ -s diff_soma.dat ] +then + echo "Soma results are different, check the file diff_soma.dat. Test failed!" + cat diff_soma.dat + exit 1 +else + echo "Results are the same, test passed" + rm -f *.dat + exit 0 +fi \ No newline at end of file diff --git a/tests/converter/soma.ref b/tests/converter/soma.ref new file mode 100644 index 0000000000000000000000000000000000000000..9b9d4abe543016b24915ee646a7151d520201b19 GIT binary patch literal 16768 zcmeHtWpos2*DVk{!2<+$g1bAx-8Hyd&|pcryQ;dos@lCHxI4_C!O0%n1{mDk-JNgG z`}W+szTCBL{@lC1Kkux?gzj+a)bl*&?6c24&xAeEp0JFso2Sc|?qC1as#d33t$+XZ zU;mF9J|lYc^#A=uN&fZkKmYtU>Yu;-x1R%A{QKkU=23nB+b{l)59w2VGjdAQ)Zx>k zqef4T{?~6ZrTYDUKK^G1{<8!B?{;8*(7&Jczw$6w;MpPn{wd`Azx&D*z>R*tbh%9}=){60o_Y?gqXPEc zH0(F0vJVT`&k5Kkq+$P8D*K`Vduzacej4_dQ`wgc*e?#)FHXb$N-F!Z0sE~1`{ila z-%4d)Az*(rV81#I``fAPs|4(?2kh6UVgE3deT{(q>wx`lY1ltWWnU*?pE1Y3^JZ%r z_McMOHwf5=1?-d4u+I=EkpJXglYo7dfc>5{>@%jaZxOI>5wJgyhJC?Q_N@c+g?zWo1>jelb9f9|J*bm{)* z{&e_5#JOi8YTgp@tgVPgIV996r9;a%BDV3pqVq`z=_?|8Wf7ZS>);zD;Po{TV>*ji zlTAX(WquzeqC-0oi>+FWPZ6L!$A8~jL|6_HGyT2s>w}1u3j_=qD59dkfA>MaXM>0V zLj`<0BVzOu5&i$-GsF0O2?=E)w3z-}M6*32HdK@#RBwZHHAIx?qeJiR0%pDw(dLMN z=lLbnI3*&ViO*~xVq+~0mQFRLL_Zagz9FAoEB?T@Ejd&6d=%mBBjC+65$w}S7E$zp2;T(}51RK%aW<3iYJmnnAB(6kM8vH&64tJ{kb>`gW+xqv4ipew zQbN`y0*2O>(D;}L&khmuLq&xC7Kgg}{Ads&q0oU?Y%8z9`MIUA_mhY+jRoXbETXbY z#J7(EMvWJ6#VjC>d#Jrl+xt9&x@{3ZOv3cJ8Vuy;ItK)7h>~#X#851JYCy0>hZg;~ zNA3Cio;uW@BH_t-5v}ftn7c_p%aw768qyP!D@z!WE*+E*D;{VD;xC@%mYxDy9}w|) zH`kp>#KVDF%sM3?Qv(TKd7cZoXX}?+@WUWMvq^(-*G24YPzDX-B^-Y{9i5*sDtORt5S=g~9tKuY@^63Q)~gURbv z)F>h#QX`zoowT#-;TJ_bGi zGU0el0m+L5JlrcGY}{P5E3AjY=S|m3_*7ko-{{}A)`0bX3FDJ|c*FHKe`7$YpCa<} zJth^9kiVmZ9L$Z4gC`+&gow2%%%O*j|56fmZSi5*M+sGy=y2`0h?nc!sG3!ezWP{f zIckQvw}4^W1ys5v!Si(@o^;ZqYgq|DPD&`5ti`22<4`}J33(1m=#(oS?R(l$Dz}PW zjNMtHfXj^KFM@=*lNg8PWz4iP2j5C4%iKR&O+t{_k2j_CIL18m(*Ken2`FD(j|NX; z(aUB*?ZE=R9~6)~iyp&vJxQsd*P|2j)uYEa6D+&**gQ}|h0=PYK!>1PahOruf$klQSlufD)2lm>qqr5( zzeHU033$o4c=1ZYLgrwP=3aE#CSqqFJ+9vr;qN2i`G$DBoUKPc{@ciXA_^rZ;RDyW zy?z{KCt9(Y*w^~LfXBV{sFl}?_)mH~ktJ;FqQ~K~0{VZ6L)AtuXl*7e`#k~4lb!I- zw4r7Y^J9sCsD%<@OXyK=gNQF9;!z2Vc~y^{S41pgo^lWK;=7rLC*k%+ zJ;E!RQkK89qERsc`92BAGDnZUR{P&1y#b_{Be2v;sQ zJdqA;%q+orS3svj5-N??V=b}ez>p*iXWWN=(4!_XY}Qf(rZ0_5}O@sSu27Q zvG#=?)?(wZZh{SYvguGPJ8|)}9uM02G2daprE=V_<9e7n3aGj(4(VF@(Br8Cm$xUP zw9k$7*_{~2ykGcK!1f~&YTNWUct=E&H%WNLxQHulz)IGhhyxO8=1GJ;)PUMeMNDIC zX6>GUHAM^4RUW=Q#SQozO@gmuWk;e?UJ64b0AN9Dpko7H#2FLRI@i52* zL*675e(lCEmlM0QN~nBQz>hz<|J(J*|4l?fvw7&ibBXCse4zh+#GNl~5>d4&{ZA%7 zBrqltTc4bKw$&ouL&rTU4x^0{OEkc zg`2ICP_DWMVip%Vv!)C8m^YUsEWN3Ri&%HP|2(wLPOR4$u;ikM7R-xtQxb7t0Hy;>CsqC#9V^`U+Mdt zQ$%nxKd$lfjTiwt7ue9`lm;K^A36hy=k(!Vq!F(>8Ssejm@Yl{B`y}%lH!q)$%ESC zl5k>*2Mw0Fu$cbOeig`H$QLRYU@OCOC_N81@<`Zu$$-7|-;!9~!kdVE#P4@oMaYT> z!+IZHt}>uH&q8Om!M%nxuZoC)8x0tj#|KF@qR)H-X3sI;{x{Zz>oK@@Jst%|dGPzx zBs3R1NWShu=KKBkbSF!!WtW{hv`ths)z+=_>A(5^R-5-e``Sbcmsa>i@4Ax1}E|*;99Z= zo5wP~H6Aq2>qaB4^F%QDR1rN2M;Y+EoW%G}LJz)2i5f=qXAQ4N96Y=%5yie3aE`d- z)sRoXh5H=uLHBY-+?Z#?++Z7OY}H|41rcSd8IfxU^Pe^JNgf&FZWz#w+@M3>7?fh& zX;ssU^LdgG(%A#eP&c~OC0{7cT3bbrm)i~K-B?1;)rlD0oHcI^&+rV_v`&x8a}uEr zG9n-O(8)0(qzWGRn;H=@Q^vw~Rv<-(Nfkxp?Pf%oIX?W^OUA1SGA_I~@Eirm#bS`I zO9J+_@M7evM0_jiLH>zuysA%*$@l)r`0I1lfF7-g10565zmtSw`;4f}TL113JvNMC zO&D*)Y2s7=!Q^$j-1s=ah)GQp3^;5>fx9}~sVm~u6eDt*ebCVN;Nvn1aDD5P?X_P>0myo3>W0?M9!VRdG zKM`-X8IdEsgrv@_|8^IuGCqoRH(_{PD{Ad0FX1_+Pcb5j^{3rI8NIJ5n8}zGW@u5n zcO1rqCZK2qFY+}{#5Tr$`#3ksG?mb{82c+??DG6Zw5I=-kK-|tT(n{{8Cv=u+{S?O z9jVP@fMcYUi&G=uKnOv`w5$0~>y6$+iY|XkjRECB9Q^}DA zjJ?8`cxZ%$d~k4mV(@(@8eKF(H`a{!a~4!jWDOyXboge(l#@Qx{3+w2PDSZxBf6Qi zNU+9X^yPSj{_e+fp9m+Oy))u&CkvKOv0#fLpfvr@374_yiVv>p3Qm+Xp@z+fYvXk| zb}|mv7sew79^8o}4o13AZ>}3pdPz8(QG_*<9uWhLSjxCNu-=cR{Fz~^jLl2P2gzkl zk5538%FJov&yphKK;IlNWMYhQ?VIXb;OQbjVjP`k%=qv4aJ{>Nt5GHlG8?h;ExFk? z4MMc>Xt2P8In@&IHJ=N8XSy-94|}dmA};X!M^7*!hPbzKxF1^PVTofhVwdy$U$8FK zPe4Kw8A)0AemO;~-tNGoTr%Egv*J+!3w{k^@5wy&^pvsct`A=)Dj0Fh1f$Og&r=<$ z)zBhmWIU|1JUGNSw4QR})D$;P3?kk!Zt8LUYcxiHaUT=v$HM9oTECKECf92iV#MW; z1axdGBRBctE%J$u(;OIBN=EoND>gkbqnS!=qX6r|2pP#-e^=&yQVuf~EHI+S1s&=! z28%R}hljZT@pwFv6elK(a-;Te36bQjTYgFCK|I;lR)T@?JD9jQKC^;y2_lZLrp$U0 zk4N3OHu9s_)Ck<{Uq)Ax@okG0TawNA6;F;${#iPXn0wKOYEA|Fx|(r+l@Vc|bP$4d z(3Xiu?}{EQ_r&A-04Hkqb0cUp@s|5}l<~iGkr7W?QqxgbKlwAMkb+N1ToY@`^8@jC zHAY5gehKd90xEy9V{{`KyEEGGo5>7;dPNX>_hT*@(HDHsu2N8p-%s0Ogx0OYni@Kk z&K!@1*L0<39R?zr=bN||CSoDmHGHKoe!0$ql{^&pwmJTPy9ygm=ur6 zu`)8JlW_lxfW_>cwhv~XY_j1%88a&C1axQp$wb^e{HG6brxXO|wx9v=c*Hg>W^Z5& zq>z7aawGb^AGH%4Ft%~y>vReIo(VY3Ubi#(ioS=0ty{ckNc_=7Dmc&e8+ID8q*Xlf ztMqq8g!8C?;7Ix($r_Vl!-{lf>}L<(h5Oj)nv6+De9+xd@V%`C*Nz!+D_o0b>vgct z^P`^Z#{8XrThBYftF9eaC(Fq8(gxFI6SBt&$V*&k_Eg5YJw8-^t)QLOf;(r3X*LZCmeFBn zoS(Xx8yAht?=S~+_1(xGO%3g>fI;*hecFgv#@%~zl$P|57zJ-u5}S!L%b)vkZ<&m8 z4_K==3fPw2j#xoPN?|+f>rJRMnH;+?wW4=2rf>CO##h$96&9?%WyGJ68g$&CL4m=3 zY^&u)aBn{<-nFA&Wj7Ym|DGr8&qMV%dBKP)LcoHDHmtMA$kD-$V8w(2{RCvp!JaV_WA-;6wuPuDaKVC;kBqpmh}c&y4u3^( z4MA>HF73yWg?#^TH~jRkF#c+>{;%Rb4`Mycd(i`x{9~@7V9`1exruun;{7lt%h-EV zM6{P01dQ6pLZy{J4IL zyycY-g@@VkKGcoObLsD=fDw80nEQ(LmG!sBQV(tu2R|n$*uR1LCFA_-BtN7S8Plj~ z*0&4Ty2OUnJda#vJK8oiq1k7~cNP(+!xcEGZw)W2!d%G;BkS=mVt=qd2FY3c=)Kd0 z@!NeEUD1xOA#ONC366Bk=|Xzcc|d;2dc1wS2i>UojN3x~#$NxvmOQqpANLQ)*l?J7 zrcuCntqs}N$#}cgjusV7*qe`MLjIA^KtY(y{I9EG%pfa{rIYdFiUz00#p2u}_G}Ip z#%X+5_{s)nHaGH9qq)QSKa%@DnDsORKQ*(^w;RHnf1pMp`iP0_MZ_dKIyDj9V%mgT`f9%k42diKAafk zLUZa6IAFv5pDw&IvHq48F*PIC&%K&U|JJ-7WM&;Wb6!Cg?*FOX%=245th`G6xhi7S z2-cdKHf-bjn6KF}?w*QTkF+?==Pc@?pz;hKN_A3Ecd-@Kipn?=qs7knIMm+h!_bB< zTrc87PKymyKD+SQLO)!;!u;#;$%t0{$bWCU@s9rE?kQ-qO~kM3GIlYi+c3AgG5%Ne zpkAKehO7L1;+Y*$tY+K+%LYNCl z_q+&-v|+VS0$dkc=S8G(%%c_N+_Qfld<5FP@i`5S%0ekWWPOJLMdY8`kE5^~*2AcZX9gLI+sVq7Vdwg&I&dZ_n}w@1qa!G zRSywxaK07Gf)p6bJBZaP-ks2*9JTYuOaA*jnE5@D8haV?VxIr=A}S^>6|rZ$f({?OxS2&kg*zg) zbJk>LZgtD0U}-Z4+P7ATn>uXaK6>XXm_+?)rd35Fasx*p8M-(DMMHJyN?pV_!iju^ zytp>TisQthM$6d$aej54_}7ixJz@wo(;qI(=}yhFyozt+KhYKihMQhYWgHX_B0plU zdts~twUyo)Ru zp=GUDyp&qx3JLxB`+LQy6SKdlO#X2r*@ax>#dW%>2=lNvgMwdHFItpQkoOrio_Xx` z+gQ=Ogn~G(e|tt1-hSlRJpU}a6|8yY#jzyj@^-2DompP9xC2C_&fHo6NZr!lvB`~{byTZ?bq5? zd@QA)CH*fVPSmIVQHbY%j_(?GjoQ#s75AOQNa`>HrjVDAe;>Nw#h7yr?ACg)_=N@K z6I>|0Ny0$#mf>YMFQETA#Gf%UUDz>Eg0qi`5G!Yb-xN%3>_tQ+1qn|?)Fq$%T-b`2 z^gpnR1JY#$Ugo&Jx(MS*1>y-Wnypr`ZKVxcipl5~OU!DhMLKwKcAEnQdwHI&)>Y7uj~HsCLMPy4?Rw>=!QI*C5PGUKPjp4e?-=+k(H$F6`eX z!SqHz#R_`l+3o7G(&ma|tZ6cCdsZcL_5K%|LlaC@M9@gUIa|^z4|NXTc zC^Jt%8Dc`w@*>jF{|6T}u7fHzp0y!I6&b}Ya=u7?{I%YTi$fipsjq_ooG!SxRy1ny@B)R=_>wA^k8Bq1up8<+lFXy=8OfwRTOlt;lPGC z1!q~4w5d_HP{Qj1)#f&(+6 z6l~Tb++H>Arw1*d6^E!}Va${>T3mOk`Vas9W z8F};Y>Uta^&%QcH12l+f#{K_dN6Yc#y%{VRNDZVd{YUWJ z%2(6lVObe>$bYA9a^f-HqrO7?nIq!zW^$P}9%O09`2Wni5ahF6Q!L2Vm~(Hgzd!e7 zSYH8sSpPH8e{6j(ik?$(D3=|XYsy$!R6vUr8q}}u#mw7wOzc2@^VW=seOyR5#rS9a z4XVw#A+dT5*FDJQL@DA>MZb!1(?sMtq9THQdq>vcKe_)mW@zwbhXob6{;J&1CH;tb z0|fYq^sk!F|aLhp6j2@xLH|4 zkNEsDPWGq1qH&@f^DncFc{hqR{W|L_b2gdW667Dc`WpONV}Zo=56tI)p6_*}set;# znLntQ zn2e%~Kifnn>JWcR(ErlDyc6+G#kt3Bbm*ht{-2x&zlcL9?;~tzrXYPz2X03wD4I>c z2=;%w>3>oNFOnaq#8f+GP$LO@$k`nEPjZkK>DYJeeC@)!m1cBl;DYA_@3F9MMOM-y zQxVpG*5C7Eoai%-^~b>))>B00>?Y(m<;J;z3icc%9xRJPaFPXa^nWUk1J$?>9mp}G z$)k&8RpI;KK|cDgS;>x5JpcInT0G&sk^65vXhGc|J7;huoM!mzxiI;xgi(z|WMKX2 zK>RDo{4d_miB{}CBU!^|b{8?Ugb8PVbL09T;{RC@eS5{>q}76TtrYB}4pXd}f@K4B zSWO(U2CK;Zz=K6kSTopT)vr&@vZW4dr)iLl+T-J?>{kxDkT=GRG1PGH(ErEQyidUX zXI3S0l@^=_H*;bPdC*Lc3L~{C>l)8Z`WZk z@yE@+`}gf0l;Qf%mbW9mrA&?`V521trB8Se*2<2~^XY$p8P38kl%@a4^{9WR)8i(4 zr3+2if0lK^$Ncv%U@mo}{~jj18tBHoNeZ6crS_RQ7Gos~=Q#?>Ub3S#{ZAiF{>Azq z#lBl!;=zfRDyCGk<9Q3tXMUqLR45L^_j-`7rXA;vE=-^{_j@iE=HK9KndiTr{=d=x zSL!d7N;|QgeMsb5mAwx2p0OtU(b~;>5(?I4q%P=*!P008$}#V699^ z6fUkJUiF~VGZp3NKT8ApzaXGUlQ``E&4ZM(cI+GD!hq&xl*s8q1pSX}!@lVUbwFap z0Q#R9;>08N0n5lC7PMeZjWr=ZIeEGnoM)6E{$z|ny>S+-Vm_=(v7_1-p7BgA^0%Tk zQ(DEaXb%d#RgsZv{ZfnlFYi>1oEnFt6$kc9~shP5&jhZ$f&`z6%on zS`qhuIK15@yzu{PSGN~sfa7( zRQwv}L7Vp~u5kS)SbtXVu9R_890pi8^ZH@K6XqStnvo|X>(71Gd+ysl_TSrCe=idM zAAfP+3;jnXtN+@Y2Jt3TeC5I=uD?0s&$n|9^7OM{*=W||)pnHoD#O`RivkTrq_Ahm z-k1D}nANL^9Zfp0rY7?)8SlR=A@A(?%!W$E$uF~;(fGR)RUh-tPdoNB)SuhulyQZ+ z%jt&>sN})QZsI@Jq*&hzAC6BVp_&HnSoY&7j?K@Z|VfqBev_Mx?jZ%?_7)>Tz% zH;l7SD$=*Kqi`K+G`q-u&T&3I+k=;vZ8*&y`~>~qdg;WxSJb90Vw+fnHQ=R{jHn80@(+fc=b zh8|4&PVU59E6RC*be8&JTP;>j@ZbVDVX?pW{hbN>pHhQ=#y*d_O{E-qjA#7?`>w9L z9Oy8f_s4dscwU$HqriJhNiNjp{+FWv`uk?$Ruc=tx&A>hb{wV-uym>hWm^)PTc{{n zm0Tf+nAM)Op7B3`^P)aYwMZW3!J6GRgnw}&oSJ0=&XCqVr~XXccp>rU4*l=t`46K1 zGQ|I|J)AAp<{ifKd{^ddgDKP_A5#yGh=!-71=g_&mP{u8zLs(NWgKIjcMaR9IA56i z|5e57&UToHzg8#bWzDpB(A|S78*E7b(21$1xu$zgs4pe#$jY83mmX%;3P1hFZ6c>( z{}s;oYgm=@wudH!Om-oR=RcqLv#88;yl-qlEB3|PM%j_|k&HeyH0W7JM2?o^eK|d- z^i9RpzN~+Irsf*)ZM_D=7#GDB*^u!X^ZS4alP@@t#PdHM%o@-B_c~)`Bj?$@9wxawKZka#q#mmN#)%J@EBgNnp|3%UBSPi_pQ zCiZfG9eJBD|AQqY6xHB%YY!^eZOBG`S0velne;#ZEAN&S;@pz!|Ffcur^Nr)OB}>$ z33vCZIKbZcRgf9YTe~oi>&->}b#MFQ7SWX~$Ujx#dN~KUC?n{N2Ie%KZp0=S~C6_m&*kkUTtT6XHGd!a0c<6{Z~P14D3H06*#kG{!LiyKm*pn zH9N@Rs7-VYGvi1d7p61*GiBm^y47uPj{JEr!MuO)ja zAt~fC#W-&(WrkePg`(3HTp|DJQbNG1vgH4ahho(k|5vEP9n>J2alWY+c`w|!l);2f zqwP4Aybne~Hzhqc|N5N`OtBY4c|>poQX5x{1zt$bM6<#7!#?#&1S61O^Lg*1H+jIZw{%{ znyAgxF=Oy&C&CzS)qk>9B|C8|uLYGRu>T0Lf5u`UB@5^naf-mhbGpUF;RjUN?GYF(I7V{M-JV znUG_w?xjJNZ+;YMYs2jsPGp^6!j&aXd}iDa&Pz>)ad4<1^+m?LCf0!hoh1xDp<>|= z-l^`bD|P;*W^6ZP4epD;r?gk`o~Xq;4|}J4OHyo ztnoPSX@1$^#L$_XEinG~jN)B@Z)Vu(|Mm?VvYum)*;|V~JjcRARqW_2|abpC@rP#r5B3{!e(p zd(e}4Pjo4JCsc)+_t{n|RQx1R~O?M@t{?y{yZ&#pqPP(^H3%Mt(DnGhs8Ig{etjqz83`gir>G8#wn&T~Bn)-n&OAE1VEn)j;b zo3UoF6FoK5mcQ}-l_v?E@0ihp_4r%74W0wMXGMJ3R)yFwLq)y<>=lDe7{LAS#Q1kX z#COhzD$h(n#m81`EWo?Boj5a_>qG+mkKz3j3-8|EC{6u6O2QCoDw!Fx19<*N4+)sE z*i63TL?_Pljrk-zp#LV1%^1RXjO5+Vqx7F`y9Srpf8^o$XUo9(K^_wdYwhUWQAXhy z-WSNs8S~r(486sfP*x|hR5igB%lbqA&6&47cn*!J6*b~Kwh}dsMCwn!ZC4SsSHNEO zs>`c65faOJRxkUU{R*y#HKlf))52vQlv5$oR0(GiE5C8xG literal 0 HcmV?d00001 diff --git a/tests/converter/soma.ref.h5 b/tests/converter/soma.ref.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ee30512247eeecf83dcf39e82a25218026e48572 GIT binary patch literal 23784 zcmeHt1$0zdmvut$pdmnTcXx*n++BlvfS|#WRMmU01b2sSG&tFfTLX=|yIbS@ zJEYPxU$5z}*UXx==KE(zujZD#dgtA9&)sLAeXgNx%huVm7R>6G@|QWYUwXe>-(RNv z^Ue47U_k08Y46j%AL4uA?;G{LaYee+7yf=J|M~fa_&(3=yZ@iB>)E|kD?eVDcK@`| z_cOm68B$+mPW#};_R;QmFaC?; zU&U|RfYcj~@x|re#qWbGsn`FA-~Zx5{D1zlzlvYIFMiUd*L?i=?=Szm;Ug=D;(&WDfPNZzWx1oIPfBS>h)>s@sC!1y!+qofj{Fw z8ZWZ@;wNqXkwYg3;vVipK+kPFK*NB`{y|RPu~9j%;kTD z1APNiT`<$PzyA&g`j$w&zW)Ci&;6L_zxTkOaUj(9WxOwb(ypB_am2{M5hJFiy^okM zV&u%hlO{$?h#EPCpN2e0x<9JC`Z?3S`H#o6 zJXPHn-)Z;#bG-k@zy9C*slQ*QR3G~s;_p}f`(*vorG7BNH}&Ga@z3uASP`dOnY(x~ z*4!z7#r^VU_RFvO{@Twk0~uS&_vzB5-Y~Q8`T+k26@8b~^o`$#ARog^`OWs%O^0Xm z3;q7Nzh6ebV174kLPS(b5EbzE|GxGq^;62VEc(CyoBEpe_^z$5xJv!3`1kMs?w>X- zx^(=0{XhIR^?oV8jqIIzhqQ7nZH)XQvQxj``#15Jdex7Ae|X@B2Yz_qhX;Om;D-l( zc;JTzet6)A2Yz_qhX;Om;D-nPCp?g%GkmZAeL{l2ujB7ey&>hV`1gPRZ496ak@8K7 zT#%|jgnqx`uXT^MsRB!yZ1MeSKfgcg9)5m;@06d-^UldX<`{VCl*B^h@VN-Wf8t(Yo+kXExrs=ND{=l8ufqu2W z`7Zctznt{>C2gmsy5`S*`R~wOQ?=3RfB62#iyt2NSNFhd-~LIT3Nn6WePe6i*xom; z@{K?H#?!v>yl=eV8?X7s>%Q@>Z+zez-}y#=-}#vU-Kn)VMwM?Ae52?awZ75n8`IV&Es3P{H_~v&_qfE})W3|r(btCeVUeF-+A)CC z1DSq)zP6iRHsAGW?ZF(r_i61CU)#kmx9{^H-L~%$? z$Tt@DjYWK8QQuh1Hx~De|5`)8(A4L7(ag_J!vEy|?S2t)`mu-_*F`*TE#iK53AIb8 z(EO!{&3v|~JQA|?7Lm1*hz&1P@Qe`f;}vWuAJ?TK%1MJ$^uU{HS%6$D=QR=|6Wh<<|wd^#y&VXIBB!UW;gTP{5PC5~?2;k;lM$))%p^rUHv58Iq$O ziSTd0d)JKH`)Om2|zlr7uJjzAM6WR>a+AJ(BHB zB|M+2z}E*N$`2B8y_JMjE6*n5Gw<0!g+u)XL=~5ixv_x3wInn;EW))_#GGIe!9T^K zjyf;uXOmE1Zwxk>-h*FINWM3enl0(F&cLGL42)Jkz5KA8_Z}7KwxCV7v zBW{RP?jJ@QjESw?)iaFQECd zSVRu$jtP|{4D<7YoXw28ivIYGYq_zzfR=kjJlMf`XAp6(zY^1r3&>Dk!bh&>Jo;?S z5)-~?Bq-J^Fy@MgUG+<$L7apmucn}6ei`kLs1QC`fM0}!$H7YaNy68wBEo|tSb6`| zcVe;i{&<}1Az^Hi3{#>NTkcH3h^Hcs({J55r)$qdOb-&#HQ$otE*}KE86ly{eG#3$ zh}aQqN3nGh^7t#^x;rt`U3201TXAG7QT7&+M3PP5uBHm~vEa!7A z;+pmP>O#R&5;h!BqUH$^tL+ksU6fEbE*jl`GvG)K0ZH=(+}kA~WXvqIDX50b`%O_x z_`RA6KheHrl?H3P5+V{kc**%Ud#OQ*uOjmDImYLckhi^r?A#mc298I}FcGVgxrgpC z{!2*MzR`ol?<7=SsKVtVBA%^wqDp2pdaGlw@sJV5o&tt!5m52E1lPxLc-TRW&ZQ-M zIVPcak`m{CjYYjY2ISr+p+n9%wC!$1iClX0VC+s81zcb(e-I?h8qYW^tHU%i_uwlD zrMdSHRh1B6^x|a+H4bwhx@mvmpm>z6rbhioG3a42q1HeFpZ5vKl}U{u+aD%ZSF6#1 z`{fPqVadd2z7dP1)6BT>LPF%}IMfcaB6^4s`H3Sl7&m<{%D0~KJC(8 zIODf7?^Cj39I|&-LmDq2dB2GE{StAzuNrM?#$s^=GxkXW+8?72s;JRzi~*+YYHa8) zp?pa-lA%Jt^;k?TYD3p{TCC_1k15q`$X?WpsBa?9c?3LTTs(a)VIKEjw`Oj1*eqgO zFEy@Q6XES8;mO)KJe#3LA6^^2TSS4RM7-r3x73Tpj07__5c^u*5%8d=8a0F5i2GfQ zhdK$HJE?J?w1B=JVo{}`14@eli+_$s(gZua(=4bF!2PjMK;%3LF~!uVyH>=9VR5Jk z#=KsQZ5Kt%=RSW}J|3QVYFM`kxX))Z4o$?3b!wEWU`Sr_%8Z6Z1myW$K&F{${IH~~)fhosY`;7ft!_E-Al6Kb;o1#%psmh|<+Q(iqkxk?GgiBB%}$A^(mDaX zxyOnnt6``8{oDsj%<%|2tj2WCam-I5lJ+Jb^fV?vnx;kzgoo>Z|`U!C4bix&G!@7(T%(nz|*e{`Cgc_@eEqezg zVkqN2_^lc>h+)$gNw{z}9^+rA@s-$YTg6-vn1EGJ)i4(sgVkd#$el%nB3X%xC)Bvx z#)~;N4bGRLzmBM3XfL43_E`8e_n_M&8!l`~KuM1i{#or9!+k&Rk$^1+CDgL0vG1mc z#xE1`m~jzXQiEm8Jz;w#)X1Fxb+87tnuwUp*v#BD9xDrJFlg2^+;UjZG(w4MIhhwe zsqr+)gW-oYxHVdhp6}GSG>`c$lLCkHdT}qn0ZmXM3che+h{KK@nI%-ZB;d=h^#2w$ z@_rH#-*h(Gab2SO5+7*44{_&1s{~YOLi-bl5AlqNgg7(`(;%`;3_35iU<7eFp$xyP ztHI#@9(4SuL7nw#ALCoYpN9J_lHuueW35%|& z;ULys={p-Ovl8nS8Z0~~B9!~$^uz=l8>B%a+AozLqWOIjai(3&ZQb`yCh<3fEtbT37DnP;3I9Hwu=aC>cwTgzZxxI+guA8A6MWV?L(zO z(HtJ^3)kXB2MzA?8U6g}m)ICwPK-lx1{Z3LNyO2KF4SM_zyjJo^-&;yAzvu3fu$7J zq2z2_%`IWuc@1{aesf}Zs5=39h~KX_iO|U+G;2I~wp@d1Tnm-a0_RHRyvic_uhU>m zZVx2A7QNTR~oBawF!J24@*#az_hJj#6PyLt@4Y z4MOXBkn^$@6}oFtfIfRkewU|sG(I+n$K>ZOWb2fOag$u=Q`L!~g&A)JL=?|OA6qqO z%UttqU?M8fe(ynAR3;t-PEupiyad!QqD4*O()by~q(C>S1!ys0HP@iB1&8J{$J7>) zBS?$=Eg63SI@H>rg<_uu#XAb9JT4m9JH%sih#RK*iTF0yg%e$z$WK00GcWCPji)Tp z;6QZ=^8*sGx1oeK1}z@Z|9fp}oHHfhabqp4$3SQ+@M|WXp~^yX;H(CGr1Fyt)mNy!A^9kL%vXyxwf(z z&$ejLvyp`ED-tlW8FSuBuHi|}X|)=aW+p&CK#M%&L&rvmkjlH@ZK6fkG#%!>HUr5j zjISUfsEZb*W_s{z4;`M5)#2lpms2jtdC*V_I7xIpC;(0xCOg{Hl z#$T^f8gy$x9B7|_z8xeK*{ww-=K9yasIhhwb3%j`Cx}md2a?xqcj8??Eyg#N(eHp6 z`EIFjvyO=86Sc@?^guz|1CQvCFNYRca|yUKEgC!8#G_^lH~JkVmOXW1ca#%BoPSaY zavj=#cTa;FjU}vq9*@4A8GqNbc=L-0!!0$wbxA;GNsEX#BHpo9IN!sG%rmr@;;)D7 zFyk$AY|YXlN|^Y}aULX@b@(|%#?9hdcz3F>FnbKvHjc+q=5~2*0fTIGXJ#Mr$f)nGNv(Rg{ex^>KThs!SN_u-i{2zPLS1#EV)EyxXNkj!On) zN@qsmR_5?v5&N!au`=0%{Ev0GP*V>VVO&1 zXIC?dNC;G8KG&*HcgEdYFJ^Wk=ZWLJRuUVSXEu<3HOQmGXx1aYm1F%AX2;{#THNYj z!jg$5Y?K9*r2U!YbXb4U14lI(M~fR!-J-?i2o(+=i^Y|BafpTsH^Yg8;SSWD<;24t z5)PynVa}jNSbr@RG4A%R@uCUGbenY8u#kL^T;@bXJQ`Qxo+kb*EJP0U$p%dZ#u(?m zzMcuLP68yx(HX{!_ofF|y2`i|X}|!Z7TaEtn{8Ggn=%gd=ejVnYCJyXaiI4!Cnoh` z&6PpKS+4)cv06kE_m&OyLdktt?63|oOSt||nV0ItBfhZ?iJAC(IYg}3V#EEMI=snZ z#{GOId>g>pll$1!U5DkjJoq?H#<0T%XgylE9;r~Zx)M3U<6xfd!hXh~`M4d&CpvL- z0P&7-Q^0^L1a=ixGw78Hh9_?G}kc)irI{8HV$u{&ap+mVb zW~_f`L{mMrjeN`t!*odE{5x~+CuTQd?p!UpomHVOW3W(@IJk)W?~cSFQMP032q$U{ zl@Lzey78NYZp4$_ttDs}zXOSj5gBEai5GE@Ic568INa~Txse~ephn}3d&M(=vl#qC9XJp?q^4>K28LT zB;L}WhZz5h=4%O6&@iFo5~{G2>5BR>nR0zuPqC^x5>;Bpq(w<6a?OZL`Y?nf|b}wV+{d zBYLy`6tjs?FVmqv<3IZb88OHBTwAp$*;j=Hb`>(;^5W|yCvp{!L!ZkwczQXpcr5ch z`Dt)|=D$T+Y~p@=<>x^~>L{Zc$mlRn#6zzZTgS&?LW~X>{UqEuDPRF>r!50nCmSr- zTgr$EDgj-Ye=-nv5B%yu>~R@^xlE`}JRY`Li5Y7d1Igsy>z#;t<3+7_8?>#Q_&7yE zpT`1Du-5HJzM}3XVbex88W4X};WEx}{+exCENl^nyn5QZD8hb7KwvoS4`+@^wqU8B z5qns}ccLFVT-IUyK@U_nWqfXJ!sWwSTra1@<25Q+XM0gs=fs?CUYzS|!&vGdq0=P% zLi>BjzsBy+Vjgj))hsupo)XH9mGOG1h}q1;Eenu4+sSFjf9mZPkh6{zmnP_t<(UPB z3kGD35fDUNX!=Np)jK_?^g>1(r3p7r64NXSCl|I#jtqJYFhb?mY`un{>$D&Wb?U zfP8%fq|eTpF#}`vCl5Af(<9$m6OP^2;_Q54Uzu3^7REUQI8mvj7YFC@`O7)srG1(4 zSCjdFIsH6<`7r353wrX8S+b1zt3~7@?zM~aLYt(+u0tZC+|(!!TCgBq2i;UFq6Qjp zo%Me$;?IsC8MWtokeE-8;(lg$XkT$dfpQsQFr6M@oPQkeg_NwrWNMoAtO7PIv|t6-Bd5`d)=dm( z`kwKfNyLeAGVIj1hL+aDSkVkE^YJ%gf1o!SNtwLpxy^xyEgp=lV8zF5PS`{VHb3s^ z0&3L0OMc0Gyd}bgF4TO+Y$ShUt$#;J9^1r=J9~9ldw_bTRzQT(f~>1`c(uxk&>@v;?TZtGF= zz7i*RpZT3+RGR8Ri4Jm7vxsk3blA>4-HLm=3*&!zH|phiEx5$@M;}`eNuKjC zREavQKl%)iu`AMp0p0aDyw!{%*|?JQ=_E<)i6|BEL=}_I}!58M^YUF=K!vth`Z$|5L`&nsidcn>PP(_uLAXI~uwH_w={hR@TpfDOS0 zJ)G1Z8j+V(nJMFJR}V6e)8p~)X6h?CTqXa>-Cco`BR$x0%8p7NH*+EVf5(BTb0q}U zr1s7D_omlj;SdRPhEu1j$^L<#9>vH%%T<>#x19&B--&;3$^WRQ<)m+izG6NsZ9~Bc zdaPF|afSSUgi*$CO+9!X#kwTef}*TJ6Uqpv_C|r(T^RQ(>}WR8jjvbC2t4V);`tIf zRTq&vuNsl*bXdxKc({oZ`x-FM<-8jI~8QD3VUbKGt8=vI*EX$Be}RGPGrF z#A-cWA623Zwett$9dYHzKLtHzRJ9;7pAMs_O~;>BLc!;{EZDIk%#F8<&9cc3WSB#Z znf2c-^4}+c+~32gv6mt*=K4P=q{p~LB6dc|X!p*IYnfz}zbRr1drd~}tu8rbENW^) z+m?FbrV1PBNB0~V%LLJmGpa~n34 z(BnM$#V6`N2Up725Xd+)>hYzQ1v5GStn5c!`BjP9K_1L$Vn?5+#P@b)_Jml2ERt}q zIP*61&w(HvDv*!&IPXBAff5Qe)8nj{TJsX}u+46qWS=FaAM$FS3Jt@}IG>Lkzkv+_ zA@oyA6}qx6IkZ(q@h@(yvg$EmxCLg;zaIDHk+({${pv;tc^6qSf=ipRU=g**r4st` z``bmS6SKakME-F#$$^~Y#kD)@5#nNP1{vSXZiJSS5%icE&urHEt;}dvOhzo{za_mM z?mpz$T>nfvWUPGb#^FTn$r%=eaZZk*tXIw}vGk4`7hl;BG{=R{Sq z?0nRp=>NOa>=iBt%8#f0ZhE}3@jKSBV~3Cvl#$Vs^=E5h?Z;YXyelE2IqlCUPSm6R zQGn}zn$H@0ncC1IJ?_|vk+OG|8ny zj=>UsYvI75iPWX~>v7c};z~t5ME1XKasEjk*dO|;MDkY?2Gah7Ha7HLE93BX6=oAh zr(NM*S>i^P?Rr$*Z$TaUe?9Ak3wIOWi5Pc4iDOSp_(cDEYuQk0wv1B5gu-P- z_|g7b2Q{vJdaOHTLH5cz6gkKKBK7eXYB$aewy~$i{1$D(&{+QA3W>CuM%FT>{;w1IkMsLXXIU$HBpyh6=^T&HeB8(QjRj3g#5XCCYK zP)4~JH`K@VaNM@Q$oaP*UmW+V0+Hk0*hGI_+)v$sb$h8O2b?>}H_4l-a{k-M6(%z8 zE_-1|OY*=~%wcUc>_1P@tl8I4Bma4r0t?6)I`>yU#vWO+L7P9GGbgW2jY)2{+WLRYO!xftX|1^ z53tx#f;d#ctH+qhB61$oBaC%>d*Hn)!75K2#gbJL075Z~wA7WlV0bb&Gfpp~k zMH&C+^{5zZrS8spoO{zhw-V_~G4CF=B1aw8oo9^b-Re;ymue6E8{1k@wW{6al;Qg%0L zl9R43WOvGzFPCsy>T`w7Teq}%SX)JSpV&y{qgDC zNV=;hrdlzT8cE1K_U6cck^) zYR4$@plL2WwA7jnH4U)2oG2GgZ25(H*pnC()|s%QvyAHxtQcCHJY=d0AGnV1v+B|0 zID2cH{~Y?ibzL1A7ZNb^Y%BsUyRfc5=fBPYPoxpoim?B6nR;&+YaHrN3uyle>%UhO z?9ed}bY}dwOYwiG0kx=A?i?jUd6oLVG6vy-2?;$|YuvV?a3DFqRfU1XA1CYXpSQSB zit|5J){401I&v%l>rAmIdDMlF7FKkeL;L-VuorZoH0@8QOa0qVjccrx&NgQKS=tT{ z_rG^8_fmV>?`FXB{!ZK;FXPcIYM&WnFj_LPpChC6c`I7d{*;mAU(ElJth;p!T{!wo zkBL>ScoNF~%um#Y3dCaQE*Jc2SaC+{z*uT?Kj(B{&QU>6`t@u1&ittr+1;m zV?D~yex~}ge^x-D#VU+GezZR=n;rLA z2P`Ism>bHR8e>3Sa&o_^>}M1s{-lpa-7zLC=YCk7Y(>=%T;pj<xY`Ge|e^A_@r1IBJT^#VMTBU2ZS<4w8`i|OWH3+zX|^Aedj0swIJ?)wP73A zzts8^+)e4}7%-K!x|(_BeLB{wV`m{lxCxyZkDsA(A?ew+ano;z@!^KZuZ^K6@m+R|7#M7{jOSk#l6SU$Y(d4Md$R+=x~v`%ZYn7=*ffS z9mIdm$-B&e9B0VM#>rUzg7xRs8EDepgl@!xe6zX7S%=mlzCGeRT2|3hyJ4LDu7`hf zD+<=8Mzfv#=QR7{GhBFf!GaU4!H?4Z^=Ed>eonoc_*b9xe|pB>tPbqEoVDS}4C4PD zJ<4%T=8Xn~ZFXQ1{a-4VgdNwWqgE>u6n$mrtyWZg%RbRY1;+AOM>o)8SOXU(eIaH6rRmc?rh*@o! z>ly$3*e~kUM2Vy!F09;PLAeihl%r-@pFN~iPpCgrH=alQxk>xGxc&oZzZCI5WG8!z zwRnc{44;*Iw*Exwkq@W`heg5F+ywJz84D*6e_!Zu;#n+Xoo5YO>2anY{r^#q7agrI z5P!{f_RE?oakr}rmDgI}f6tCdCpf3ucIcl;*qWI&O-?n8%oSeRk6lkr!}_Znt{t! zWA6Vz3Gsy$xY5#u@>UD7klz(bGGH3*&-uu+Wd+!`g<@Y?2fi?5MGfp(&`i~lH#psSY zl(^0_CSCfJ zC)rzQe@}rf&)irx&cZ&A9bb(G4BBl+qX0F&kpH&j{_juw^|=2R8f_>$LPFUidZ@WJ z`tnBP&E-H}+Hdh$#L;<1^vZ8S0P}IV0?g-E_`72Ycv~}%4br-;S2ki$~@Y!I;$yfu(YeJ1_0(vFMdJ#9HhJMp(PbG4;8?F%-zS%*Ji4N)e5{oXJm z%HFW!+;ka>nEz&KsAFU|VKU<{>Mi>pw10;^md~ue9jq0NZYO$XGN2r_`B#0}Ga<)V z(L;eupS&p4+JYNX?Z`aVfQt+5c+a>W7(`8naj?Gu^+m?LBF2V%9VHArs>i%9JX6`t zi1O48hegW}DognFo)<>eFCNC@aJ{e)9{To*;$b6bLW=|!0aUBZ|)?)|zs;wFSgSq}Qxc|zN zQyZ2j5xpr6{-rD!OAT&7Zv&cp?U+UTZt5?E8Gk34t4B7X{yd(&DbD{6_y5?ZJO@32 z=R_9~E4kl-O7Og1p9D-`{j`qjk^V5x9-Yvkb!jDLmZttTf?6!khRpav{yC97vKHi2 zg?Yxxqr(0-ap)UlfvLA0qO$;poS8L^88HR`$k-4JSP5P!Q_4H3o-xZW&QblvL27Zcs7^$=M>L6Ze;#D z$~{v#pMa?q;&GQ}h5EF%V^<#oZdmO&Oxkgly+?3d>_+4<2vyC&|M>J+>ie2EbtuA;p=tNSzl-5{1|`5+UPLu z9?z;b5OBD2Jarp0n%1=Ab!P+I>_O*b-SLRH9UQ_vO8hND-1kzW$jQ+}{;`C)`(nHi zzm2gYKka);QI{4Hah~?y(jSu+Sy1Mn4*T{ffRy@k7IPN4Vsv%`D#uvSwwDecj|_G(G$?~xJ)QB%ptnC-{)KeS)K#05t3 z9XmR(r?1T;;V$hreqh8P#$!0oejcLzEL#+~!1^OM*FQ@-_78F!P*7<_&-OYLjOKZP ujO;PbipSvV>5w)U*?8jE5rjbDX>Hh)T{A{2A literal 0 HcmV?d00001 diff --git a/tests/converter/spikes.ref b/tests/converter/spikes.ref new file mode 100644 index 0000000..fd0c3aa --- /dev/null +++ b/tests/converter/spikes.ref @@ -0,0 +1,1148 @@ +/scatter +16.075 85165 +23.325 63004 +26.050 70362 +31.200 69687 +33.975 88545 +34.850 66144 +34.900 72526 +35.750 80355 +37.100 84664 +38.100 89034 +39.475 92789 +40.200 68006 +58.450 78617 +58.900 77837 +63.900 85165 +82.200 63004 +86.375 88545 +19.975 72571 +33.425 70484 +33.925 72571 +34.575 78666 +34.625 71255 +35.375 93693 +35.600 80642 +37.050 66343 +40.975 84907 +41.975 63136 +51.000 85284 +54.600 68152 +57.150 80642 +68.925 89154 +70.025 78151 +77.150 72571 +91.400 88671 +23.675 80964 +29.100 85535 +32.000 73386 +35.575 78553 +41.950 68641 +60.325 63698 +66.350 71012 +75.050 66698 +77.950 73386 +96.500 71621 +27.600 90143 +29.950 68763 +30.675 63013 +33.400 81362 +33.475 85529 +40.850 63984 +43.825 71257 +47.175 73433 +50.575 71985 +58.700 67038 +67.050 86181 +93.600 85529 +7.525 89769 +28.050 68893 +28.300 71545 +30.500 80963 +31.825 72860 +32.100 74894 +34.200 85658 +37.650 63259 +41.350 67370 +43.725 89769 +46.825 79745 +55.975 72860 +58.950 86325 +72.800 81480 +81.200 89769 +86.875 71545 +27.300 79876 +28.900 85794 +28.975 72756 +31.800 75336 +31.875 73198 +36.575 64560 +37.725 81608 +38.675 64523 +39.600 86469 +41.975 67454 +42.600 90768 +44.175 81478 +51.500 81608 +65.325 72756 +72.025 79876 +76.350 68985 +77.500 73198 +78.500 90012 +87.750 75336 +11.000 86855 +27.650 90889 +29.325 79937 +29.475 73304 +30.550 76646 +31.650 90146 +33.175 81732 +37.375 65179 +46.750 85915 +48.050 76646 +48.950 81605 +49.550 64747 +52.175 73304 +75.150 69022 +91.800 73041 +6.300 76702 +12.500 69131 +21.200 87233 +25.875 90270 +30.050 73388 +30.625 76702 +31.975 73482 +32.500 81986 +32.950 80125 +37.725 65376 +39.375 65139 +46.675 82245 +47.500 80125 +49.900 91409 +57.525 73388 +59.000 86719 +73.900 73482 +76.875 90270 +92.400 87233 +26.725 91658 +28.625 69447 +29.600 82369 +31.325 74005 +34.650 83011 +35.825 73532 +41.500 87102 +41.725 68269 +46.475 90891 +50.000 80641 +50.025 91658 +53.200 87102 +53.275 69447 +55.250 74005 +57.500 87923 +79.550 87102 +82.200 73532 +84.925 83011 +90.600 69447 +96.900 87102 +17.975 82762 +27.900 80779 +29.675 87920 +32.325 74228 +35.800 83135 +39.725 91411 +40.850 65822 +46.100 65759 +46.450 68807 +49.300 69613 +49.600 80779 +51.225 82762 +53.200 74228 +91.275 77474 +91.525 88050 +93.350 91411 +23.325 74380 +25.300 83137 +29.250 74617 +36.750 65853 +36.950 92051 +38.125 66140 +38.675 93175 +39.400 88306 +45.175 69914 +50.550 83137 +50.925 74380 +65.050 74617 +77.700 83406 +9.900 88420 +22.750 69969 +32.025 74575 +32.775 70481 +32.875 84022 +35.125 92293 +37.575 66375 +41.175 88420 +45.400 83264 +46.125 78779 +56.375 80903 +71.050 93301 +80.550 84022 +87.525 75474 +92.025 92293 +19.525 83409 +20.700 66751 +23.050 66791 +26.200 93423 +29.625 92673 +30.875 79076 +31.500 84140 +32.250 74615 +39.550 70208 +43.975 88781 +44.400 75901 +50.125 88302 +53.050 81091 +54.050 83409 +56.975 79076 +66.225 92673 +10.900 76457 +12.425 89032 +20.250 75176 +20.850 66960 +26.950 84143 +27.950 71488 +29.300 66790 +32.200 79261 +32.325 75176 +34.000 93305 +34.900 84404 +42.350 88541 +43.925 81155 +44.475 76457 +46.275 70400 +46.475 93550 +49.800 71488 +50.500 84143 +53.300 75176 +58.275 76457 +90.175 76457 +94.175 89032 +23.800 67639 +27.875 71543 +28.075 88905 +30.875 75290 +32.125 89400 +37.025 76521 +37.450 84665 +39.900 93429 +43.550 93689 +62.325 67407 +63.600 75290 +70.225 80776 +82.150 76521 +91.700 84406 +11.900 71256 +22.475 89633 +22.875 62703 +26.200 93934 +27.850 72117 +30.575 80961 +31.000 81988 +37.575 84910 +39.200 84527 +41.500 67669 +45.075 89029 +45.175 75411 +57.625 76704 +59.525 80961 +60.850 62703 +62.350 67452 +68.250 72117 +85.725 71256 +21.200 84911 +23.175 71349 +24.225 72469 +30.350 75472 +36.100 77411 +38.000 63229 +39.900 68187 +42.475 90266 +46.850 85163 +50.275 81152 +51.525 72469 +52.425 75472 +54.375 67752 +12.675 68268 +17.775 90652 +18.400 85414 +23.525 63333 +24.475 71489 +26.075 90390 +27.875 72617 +31.500 81206 +33.200 85534 +34.750 75715 +37.725 64524 +40.525 82243 +42.775 77585 +55.850 85414 +61.925 81206 +93.000 90390 +95.650 82243 +12.400 71713 +24.675 68535 +25.175 90770 +29.850 81357 +31.525 77777 +32.250 71713 +34.125 75780 +35.225 85799 +35.925 90519 +36.650 82367 +42.325 64605 +44.825 86043 +45.450 68400 +50.125 77777 +68.000 81357 +73.350 75780 +20.625 69449 +24.450 72073 +26.150 90890 +28.075 73817 +31.350 90647 +31.950 63987 +31.950 85919 +32.850 68492 +33.025 81728 +45.325 76096 +48.750 82630 +53.150 73817 +73.250 86179 +83.500 90890 +89.150 77892 +93.475 81728 +5.750 64995 +12.275 91410 +25.400 70123 +27.600 86323 +29.300 74188 +30.300 82890 +30.875 86044 +31.175 76155 +35.100 81982 +39.100 72572 +40.900 64995 +41.125 69614 +48.575 91525 +52.425 91410 +53.775 74188 +83.300 82890 +85.600 78436 +15.100 86467 +18.325 70207 +21.250 74379 +23.000 86184 +29.750 83408 +30.250 69655 +30.600 78485 +31.525 76583 +33.675 91531 +37.225 65181 +40.100 82111 +47.075 91787 +52.025 76583 +54.625 72755 +57.825 78485 +74.300 86467 +79.975 76583 +98.325 91531 +98.400 74379 +15.650 70241 +28.525 74795 +31.050 76936 +31.100 72950 +31.825 83905 +31.825 78668 +32.675 86723 +35.150 65501 +46.500 65823 +50.550 70165 +52.975 82481 +55.925 65501 +60.875 76936 +62.550 92050 +74.250 92047 +85.250 86327 +12.575 70719 +16.450 73087 +20.450 84142 +22.525 77131 +28.075 74852 +30.950 66177 +31.450 77131 +35.125 70719 +37.400 87105 +42.925 82759 +48.075 77131 +48.600 74852 +48.600 92178 +51.450 92290 +53.175 79322 +65.150 70797 +65.750 77131 +71.525 86856 +86.150 73087 +87.850 70719 +9.300 92558 +11.775 73483 +15.525 71223 +21.700 84273 +23.150 70841 +29.975 78894 +30.925 75289 +31.375 79441 +39.025 87236 +40.600 65678 +46.425 82888 +49.975 71223 +51.575 73483 +57.700 78894 +60.900 87341 +64.275 70841 +67.350 92427 +72.800 92558 +81.175 75289 +98.225 84273 +12.650 92672 +17.875 67330 +18.150 73929 +30.425 87579 +36.525 84405 +39.725 83131 +41.050 92554 +45.575 71011 +46.850 92672 +57.875 79028 +63.275 73929 +71.225 80126 +12.950 92915 +17.275 71622 +26.225 75779 +32.250 65985 +32.475 68109 +33.800 87793 +37.350 71102 +43.550 83404 +46.200 92915 +46.200 87795 +48.375 74099 +51.625 79262 +52.475 80404 +63.950 84782 +67.625 75779 +83.725 71622 +88.050 71102 +5.225 93546 +14.275 71302 +16.475 88051 +28.400 83532 +33.350 75844 +37.100 68682 +38.700 85038 +41.875 79549 +42.525 93303 +43.300 66627 +46.475 88051 +46.525 74279 +48.150 80579 +62.425 83532 +70.450 88305 +13.575 72119 +16.675 88422 +26.625 76153 +30.475 93930 +30.825 79678 +32.250 84524 +33.150 89151 +42.725 66873 +47.700 80643 +49.675 88422 +50.125 69023 +51.725 72208 +58.925 79678 +61.775 74329 +69.700 93427 +87.225 72119 +89.525 76153 +15.200 72379 +15.800 62798 +16.875 88547 +21.025 74438 +27.425 89632 +28.375 85530 +30.275 67408 +30.650 85664 +33.275 72247 +34.225 80828 +40.350 93823 +40.975 79821 +48.975 88547 +56.425 80828 +72.500 69093 +78.975 62798 +89.150 85530 +17.925 89767 +23.475 72754 +27.500 79875 +30.825 72468 +36.550 88782 +37.300 67591 +37.950 86175 +38.325 69168 +38.675 80905 +45.275 76878 +47.750 85798 +49.950 79875 +74.150 89767 +13.200 73086 +13.225 72861 +29.575 81360 +29.775 67754 +34.750 62754 +35.675 85917 +37.425 63257 +45.325 81092 +48.775 77130 +49.775 81360 +51.175 69202 +53.525 74665 +56.775 72861 +60.950 88907 +67.150 86464 +94.925 85917 +14.400 70046 +25.575 74853 +30.200 86584 +30.625 77535 +42.050 81606 +44.600 81363 +44.875 73199 +50.275 68679 +51.825 86182 +53.325 77535 +61.975 89521 +15.475 73716 +15.900 70272 +23.700 73716 +24.825 63699 +28.225 68894 +30.275 86470 +31.150 64268 +31.600 73254 +32.500 82115 +36.575 81609 +38.750 87339 +50.475 73716 +52.775 70272 +52.900 90013 +54.800 73254 +81.250 73716 +86.025 90521 +86.525 86470 +8.150 73880 +18.350 73880 +26.325 73880 +26.500 86588 +29.475 73432 +33.200 81990 +35.875 68944 +37.275 82629 +37.825 64789 +38.500 87450 +41.975 73880 +42.450 70483 +42.600 78830 +43.200 90651 +45.775 90147 +69.275 64164 +73.975 86588 +4.350 79024 +11.950 69285 +16.525 74576 +28.825 86725 +29.425 88045 +33.525 65760 +35.575 90396 +37.200 70767 +37.500 83646 +37.750 82118 +39.125 91018 +46.875 79024 +50.400 73620 +56.675 90396 +61.650 88045 +64.175 64234 +70.225 75413 +75.950 79024 +84.850 90396 +90.575 86725 +99.200 74576 +8.900 76394 +15.175 70980 +22.075 76394 +24.850 75473 +29.375 79077 +32.600 83781 +32.950 88303 +36.575 69806 +43.150 82246 +48.850 79077 +49.525 70980 +53.375 74004 +55.150 90524 +64.025 64666 +66.750 91529 +71.700 65882 +73.825 86973 +80.525 76394 +81.475 83781 +29.525 83903 +30.750 87234 +31.975 88543 +34.050 69879 +37.025 74048 +37.075 65918 +49.075 71103 +57.675 90653 +58.800 82370 +65.425 83903 +74.700 74048 +94.900 87234 +95.675 76805 +15.575 90771 +19.125 66458 +21.925 71585 +27.950 87343 +32.300 82485 +34.100 89031 +35.525 69916 +37.350 64862 +44.400 77183 +45.700 74097 +46.775 92430 +47.825 90771 +55.400 77297 +63.975 80124 +69.900 87343 +11.325 77235 +14.300 70361 +19.275 92914 +29.525 71714 +34.350 89516 +34.575 91791 +39.225 84526 +39.825 77235 +41.475 80464 +42.825 77410 +43.350 82891 +53.450 71714 +60.900 66919 +62.200 87578 +75.425 74326 +81.050 92914 +25.125 77476 +27.400 67329 +28.125 93047 +28.700 70482 +32.750 74520 +33.225 77475 +34.650 89628 +49.800 80520 +52.550 71762 +58.900 84666 +61.050 91920 +72.525 70482 +9.950 89893 +21.625 87794 +25.750 70525 +27.650 67549 +31.900 85037 +35.975 65916 +39.125 84275 +43.425 78072 +46.400 85037 +48.875 87794 +63.025 77648 +70.125 93177 +79.725 92294 +85.000 80578 +88.175 72380 +99.525 74946 +10.250 72423 +24.000 92788 +30.975 75085 +31.075 70571 +34.100 90009 +34.825 93551 +41.625 68308 +41.700 84528 +44.825 78150 +48.050 72423 +63.625 80640 +73.900 88421 +80.300 65952 +85.725 90009 +90.725 84528 +95.325 72423 +18.200 72528 +19.850 78484 +26.950 80711 +27.150 75226 +28.925 70932 +31.475 68681 +34.500 66069 +48.400 72528 +49.225 88672 +68.500 88672 +95.350 80711 +8.675 62713 +19.300 78667 +21.500 71222 +25.500 89033 +30.725 72809 +33.875 93935 +34.150 90267 +34.950 81270 +38.425 66106 +44.525 78073 +44.650 68896 +55.125 72809 +72.675 89033 +83.200 78667 +17.550 73340 +18.725 68945 +26.475 86324 +28.075 89153 +30.450 78832 +40.125 71305 +45.500 90392 +48.225 81604 +49.100 73340 +49.800 62782 +52.075 86324 +66.325 85040 +85.350 66141 +87.350 78832 +87.975 62840 +5.225 71393 +18.325 73818 +19.500 76211 +29.950 71393 +30.575 89634 +33.125 90648 +39.425 85286 +47.800 62971 +51.025 73818 +55.300 69287 +65.175 76211 +74.650 86468 +90.675 81730 +98.525 71393 +29.675 86587 +30.925 81983 +34.450 66872 +35.050 85415 +41.175 63428 +44.950 63070 +52.200 89897 +64.100 79142 +79.975 74189 +88.500 90888 +21.000 67078 +23.050 79440 +29.100 71984 +29.275 74280 +29.300 91017 +29.950 67078 +33.400 85665 +34.825 86724 +39.900 63597 +42.600 82240 +45.625 69688 +50.400 67078 +63.625 63464 +66.225 91017 +74.125 76875 +85.625 85665 +88.725 82240 +94.325 67078 +17.675 74523 +20.150 79499 +22.975 67667 +25.250 90395 +27.700 82364 +34.250 72118 +34.375 86972 +36.850 63533 +38.950 91151 +43.500 69968 +47.900 90395 +52.025 74523 +58.825 77181 +89.375 82364 +97.100 91151 +5.025 80293 +14.150 82889 +16.975 72378 +20.975 74666 +21.550 79995 +29.450 90523 +31.400 86185 +32.600 68224 +35.275 91281 +35.300 80293 +36.975 87106 +58.350 80293 +60.550 77582 +63.400 70166 +91.200 80293 +95.825 79995 +97.750 72378 +28.300 70625 +29.775 68354 +32.075 87232 +32.150 86471 +32.450 72618 +33.000 91662 +34.675 91408 +37.325 64461 +45.400 83009 +49.200 75001 +51.575 77714 +61.200 86471 +85.050 80904 +87.075 91662 +8.900 91790 +20.575 80826 +23.600 68533 +26.850 91527 +27.700 70680 +31.325 87342 +32.850 72808 +36.425 65034 +36.800 64716 +42.475 75087 +48.025 70680 +48.875 72808 +51.250 83133 +63.275 91527 +78.850 80826 +84.325 78147 +94.850 87342 +95.375 81207 +13.700 73197 +21.925 81989 +26.500 87344 +27.925 71224 +27.950 65223 +29.500 92431 +32.800 75541 +33.075 87452 +36.500 92292 +41.925 83643 +42.900 65302 +50.875 87344 +51.200 78255 +58.525 75541 +77.425 81989 +94.750 92292 +19.275 81154 +21.625 82244 +23.675 68856 +25.550 87577 +25.875 83778 +28.600 71348 +30.275 92429 +30.600 75606 +33.900 87454 +37.675 66265 +40.900 68856 +53.650 92559 +66.500 79141 +68.650 73530 +85.750 92429 +92.100 82244 +16.725 71667 +22.000 74050 +27.175 83902 +28.150 75845 +29.200 93422 +33.100 93304 +33.725 87922 +37.625 66419 +41.225 66071 +42.450 68942 +43.050 79207 +45.275 87682 +48.975 75845 +54.000 71667 +70.700 79207 +9.300 79318 +20.150 81479 +26.650 82484 +28.150 76021 +29.725 66176 +30.675 69531 +32.450 66577 +33.425 74227 +36.500 93548 +42.900 72279 +45.100 79318 +45.175 88049 +54.050 76021 +57.250 87796 +93.325 82484 +97.250 81479 +9.175 79377 +25.525 93932 +29.950 72949 +30.200 88423 +30.425 74522 +32.125 69840 +34.075 89152 +36.850 82631 +38.275 81607 +43.800 84402 +55.425 79377 +57.100 67159 +58.225 88423 +79.025 79377 +85.175 93932 +21.325 83906 +25.875 93933 +26.325 69878 +26.725 76339 +29.700 89519 +30.050 81846 +30.200 84525 +31.700 88673 +35.225 69878 +39.075 62859 +46.950 67593 +48.450 76339 +56.375 79436 +63.500 89519 +69.700 83906 +79.175 73146 +7.775 76396 +21.325 62926 +29.625 73434 +30.875 76396 +37.925 66376 +39.400 81987 +39.525 79936 +39.925 88908 +40.775 67709 +41.800 84909 +44.300 89896 +46.275 76396 +53.625 84274 +55.500 76396 +65.150 70242 +68.100 74712 +83.475 62926 +22.425 76456 +25.850 73531 +26.900 85039 +26.975 89035 +29.275 82116 +29.675 90522 +31.950 70765 +32.025 66918 +32.225 67844 +38.250 63393 +44.925 85161 +50.400 90522 +52.575 76456 +55.525 89035 +59.025 79993 +67.550 70765 +85.300 74757 +21.200 85532 +22.625 77067 +22.650 67005 +26.475 67927 +28.275 73621 +29.875 91154 +31.975 70930 +38.225 63429 +38.650 89636 +47.275 82483 +51.000 73621 +53.875 77067 +68.425 91154 +75.900 80291 +92.125 85532 +5.975 77133 +8.350 75900 +16.025 82761 +16.425 67328 +21.075 71144 +24.325 63567 +24.600 77133 +27.850 75900 +29.400 73963 +30.800 85661 +33.600 71144 +35.375 68153 +38.550 81475 +41.050 63463 +41.225 75900 +44.400 91530 +53.050 77133 +57.925 75900 +66.050 89770 +70.275 85918 +80.225 82761 +84.925 75900 +21.725 74051 +23.150 86183 +23.900 83012 +25.900 90014 +27.325 68355 +31.925 90014 +32.350 71392 +40.275 63598 +45.425 74051 +45.925 81603 +46.350 90014 +48.800 91661 +51.100 67843 +56.600 76338 +57.750 77236 +57.875 86321 +62.400 83012 +64.875 90014 +67.750 64236 +81.125 90014 +18.750 86326 +28.575 68007 +28.700 74577 +31.000 77298 +33.875 64522 +34.150 91919 +34.950 76756 +40.225 90397 +41.675 83136 +42.600 68608 +44.000 81844 +45.150 86465 +51.400 71487 +51.975 74577 +56.400 86326 +20.650 78152 +25.875 82480 +29.525 83407 +30.550 68680 +31.675 69056 +31.800 71541 +34.450 86589 +34.725 86721 +36.200 63948 +36.350 64637 +42.175 77296 +49.825 78152 +53.100 71541 +54.625 90772 +60.700 92049 +81.000 86721 +87.475 83407 +96.425 86589 +15.650 68720 +28.200 69450 +29.350 90892 +31.050 87235 +33.400 83647 +33.800 71712 +34.725 68720 +38.300 64024 +41.250 82757 +41.300 77409 +41.350 86970 +46.675 78554 +50.525 75781 +6.975 87453 +28.250 69492 +30.800 92671 +33.600 77836 +33.625 79321 +37.100 87340 +38.025 64750 +38.875 83130 +40.150 65677 +41.925 68764 +44.300 91019 +55.950 71809 +58.225 76020 +65.600 83904 +96.575 77836 +12.225 84023 +31.175 69720 +33.325 71894 +35.100 87573 +37.400 65064 +41.000 87681 +41.500 91155 +51.875 68806 +58.200 84023 +69.925 71894 +70.275 65953 +73.175 79822 +82.575 83641 +89.250 84023 +91.500 92787 +26.950 77066 +29.075 67158 +30.525 79877 +31.050 69880 +31.925 71931 +32.275 68895 +41.400 87921 +42.475 91283 +47.075 93178 +47.300 77066 +55.475 79877 +59.550 84141 +72.025 78149 +74.075 71931 +78.425 83777 +16.525 72116 +23.050 79938 +28.225 69917 +30.775 78375 +31.675 84272 +36.625 84019 +37.700 88186 +40.750 67668 +40.875 88185 +41.075 77132 +41.500 91792 +47.375 79938 +49.825 69286 +51.825 72116 +76.050 84272 +78.700 93302 +90.125 78375 +10.850 78435 +27.275 92295 +28.600 93426 +32.225 69367 +32.425 72278 +33.825 84138 +40.550 84781 +41.100 88304 +44.900 80064 +56.700 67800 +59.075 80064 +64.350 88307 +84.550 88304 +90.925 93426 diff --git a/tests/converter/spikes.ref.h5 b/tests/converter/spikes.ref.h5 new file mode 100644 index 0000000000000000000000000000000000000000..887fecaf981e91bf75d2e0dbbfb12aadd5d5575a GIT binary patch literal 28072 zcmeI3dvw*+mB()gB9EX!PzWFfkq8nYBoK%^a`U<*B#=NLKz#*})YO0(L#?T_jf*xV z8Z8Q9_St8j$8Volm6;wGI(YP8U+8DxKwp1fr2D7+I?K<__2L+9?S=d!4BP_l z?5BT(`9lBse52qG!~VH)%a&$j_=1vV-w-ul#|8S=Kx6db-&F)MGg9(G1769fElXb} z&ny^arNy_cD+^_kelo4$uf+4^Z)Ug*-b!A6R{p*{Nb-I8`&UlH-?-ZO@^=gQW5Y`+ zDRKY&-^$>r8$RzF72ID#F%I2V z6zhS(x!kZW6t*?x!5I~XraLIm4;i1&t~Fu43&mri!njF?g^6)@|Bxl0&oDgL9%>mD zoy?x^_stmMn{k$X{(>Mk)ZVXOKWAVd^5J104Gr1x`HoEx>h~pB2?;;<_WZRGKEG=S z^9}Hg2)2E3L-EE?Qbq;+361g7;|vM9PJ$R3&*AAIj`89|{11OCkNbV{J#jeYFB`CZ zww`_ZFH8UZv=+IaVg5&IgFjIt{EqDOP>~o3f22Rb{X{{JL2n#*h5yyI;7?Apd(Rj??%%pnH%DOlKTm<8;|M zI30QTC#S2Q9@=B#87G+Uq2@cDamP!qIbQZ!z~}_E2Ru|hB}lFUer1C6+rb|AF*Bqe zL|^E~KSTbMfi>{!k!yh8Gy{8J4|2YlvNvERaRw85qxyrN@1f&Y%p{&O{f@5p&@HE? znZyIRlO8&GUn2b{GS7*UFHe+zm5JDeZh&q9JCN^z_RW%=h~DVdf6Of6NPFxo^`Gyd zo6mCODrd=`TI3r&)cBy=knfx&e|@v1moyvuU>W>!uy(fmZKVAO{Px+h6P={?*d)<@ z=$s^tqX>QlbS3T8N$R(T_Le01(+=Hne$+mRorvq0pX-Plbb^O2UP;jT9x9I4F`w7T z|B7?49=i?5HJ^j+w0B~+t2erSdd$B$#A}ZH4M6A2VIJTY!7rJEUvrpe4_zD@kPo8f z+e6oHJMuyFg?>6bR2=E&Bzj%F(d`2fbBXi$(bPAaKZ#zCjc1*138K~&`i&mCaW*5@g4{vc+dOpr9ihJpP+=m*j1ci~?TdcI`Y z38Isa=#5TqcrtmM%sNCa)7Bk``eJK_0aiOhujYMyRg#;zo|F6aqPiv z5S^X9$hAVZdFcE-jQmmf$Duntbn|=y|4(_SJVLG;{WG-tQWVz#DVqO5DddTVuK&ms z<<)QxU4K!?jZN`8xoGHE=s4^qdFbT)&{-ZjejfJoJ=DBo_XZE0d?Eb=(fL#Cp^Iw? zcFH_-a^=VcQR^JL)gC(gTd^NR7sndf>pXP)9oT6=ez%9Ny_NnBdFcFWL+>c^9nhV~ zoq&G|emDJ|0V7gbx2c-9*i?-tG1c$<_j~C4NTWS~URG~(dU>hrqv#i+S42N09x9&D zo6y_rq2fzFRmfHMMrWrMejWNd=(hp=U9>lP==|M{e3OS-XXrIEjuz-c9yq^g56`fnGOs5B>Sl*pJg#S80By7YRKUIvTkNX_~*7G|lT|=tSiF&;jK0 zp$mGWi{~oz3c*tBmZQHJx(a`)k=yE_=7sh;_`A_>@=)^8RTsS`w?`Khi<%cY5x>ze}bR$8F)j6=6@}Ai=oRr zbbj55UzPA{kgtW`KtGMpdomRN7U+W>I=_!YcVg!RbT@X+pzjM14=~0KDRfzIQh{_ zL$AO?*Ir2bTI>~Lr^G`iS4w+1SmmM1w{6(jf&B)?w;OChz8$&)xlZ^e(C-4nGd0cu z9=drMl*#^=$-bAVIL5(G$W%O&z%1l)Jk+?6UyJ=B+Douk>Y?kmoc2u~I(`*&HT~5> z*I~DTep;aq(%uGrl=iovJA0$+=LGhH=<=frx(9n_@Yk26_V6slC9*dvzmS^%CiF(F zOZ5G;r)6ng0vhpykvS(>+M(B0v$ju3p;DUvV|JwCgdtTbo$lE z)xh6|omvlFymrvu0Dsp)&FgOHX6&@WKMZ{Yd<*_5(3j1+^3cU4E?fTQWXpb@hi-ir zL2u1gylSB9vC{y*33?Cq_9E9pdn^2t$aQBc?mh6&ARm__KZ5AukdPz46Latrx*$jT ztB_ku`zEj&ehv2O!4~+f(1)>i6npLPJD^WMpYqVfrJMGsMe=K`hgxUQv5Vwq(jtu~ z4SLlgjjIT{nD#Ocb)M5+zDRMYpnWs^D(F`19fUu5vFryH6Q{-Uvt+UClrEM(Wgcq3 zL$4COTH23*Z_(b3y)(!~<*J{_x#}l2m-)+O{=fkI&ESr6uqjvmwP2?W`ml#CzFqX! zgWMVDusroQAW!`d$&+8h;9r&}`?G0Jqdm(*XFoen<5-5=_2?I&UkZOS{0BUA>-=Hp zZOGSyyU}Z2ZshL4 z&KCN4z(XhhAaX%;eW9Od=?;ZoPb!_OGM2h4w1wt=O-DzYFdI13zz>#&;8Ts%Wo12Oo!S!_E=(-oQ==^t;H7ST6r2ESLR6=s6yS(!=M6 zP6M--%l|y+eDn*zBHC|--r%9LSGrvBxCg94{@c*sg?@Iq;&1@{pMgigH;{i5`BU(_ z!H@ATyg=g|QXs#h3pC!T1>}i`&c8h5R>EHmy#+h>709p0vC{zmDd?x6pGU3@JFkQ7 z*z16P6S|*;@ANEC{F9LlL=!-yK1o|S-7lFPA^hKaA0(}wa zi$Grl`XcbR5`obFm*u|y-LKI1|0w7B`Tu{hNGBgE7YuZge)s$Q|DV|5EGO_8|DTx8 zS0S;!-+dA2i$Grl`XbO5fxZa*pGV-IX6WF5JW1GbnXq`h@Ynw-7jmu@y?u(XZn<#h zO~Tu*5&mPe@a{>%k3NuoG41yki65CL{KZV+{l&tytAtnGEbJUDToW(ciJd^c=pW+O z;44J0!2icD5}lYS9JWpvJxchK>xF}nn?FMIS^Rivu;|lY5Ps!yVcl5ao+#l-_%El4 z{s#RYyF~Q%VZ!}OglU%wpJQBI_??K}Lu=H2dxWqMJB$9HIDQQ}ll~g@dgD7gK=NP3 zpVKQuH{oaQ1kvk=@9B}Ee+}J?KQF<*dz{)wqc;Y>!Y&lQfqBoukC&PE+%amOkDsgH zTV6b|NbPw;g)@muZMf*LdBXRJ`@#g#Kb$N)oFS}<7k-)k?j@fB^TmI1qVNaIfAbvC znHLKeOc(C!FZ|X8!d;B_0^<47mE!-Fd3!or^q9rMmSw_wk$aVSdU1jHGnv;f&la7Q zB7DC<7?&kX9WSgW{^@?vDe!ImJW2jX6aR&l|KMK6kw@OW%lcb7Q1+h86LvE0T;i63 zf6uZG8jyPi|HpJI|L$X)(^&_%d?@}iLuB_u@-}U*=uh&S{?pRMFJpd3Fn^0blASH& ztDiX7zOx7WKS%CxjO^@~DtwUq9KBj}3;q0#IJ|tl_`lyE97X(Zxk~h=;lk@!kJC~` zr!bG(3q_wqe>?kAJ9&O7{SI0yxpzy1?;ziAlITA~3T-_U3=*BdI+(Lm^fdO@yU6{n zh{ti{1~TrEMV=k}Sb30)+*IN)mi=NS`^j47Et7Hi zZ%}`C-X=^Z-XDNjtozHcGnTw4ga0t|-a#JRMgLKYWiLA*yoP=on71bUexLacU~dU_ z^4L!fFt6itWoJ11NdoI=Is45t;x&$b-@)!(*tweZ*UJ9zAN^(T750fO%!8l#8Up>B z8It=Q@jJOf^dsc$igBWU30_nzdi*Hi&oYGj$eY*Lr(2lkcJzF#>+2Zr?etSiKTqE% zJ74Q3d=0z128s?N|JtxW>=yBDKYk=awB1uaAb!^}zNhfxN6C_lT_~J|KSi8(m$43W z$={dA>j=huC{22=vCl*i|IzFl?{O}jV11{u-}NW%3-EK@6xs39KK>@r%dZwb951wc z)f-bqKgRrD#lHIpd37t}eQ}xOV$iQc{$uSr>K=obJ&$OGc>uJFN3ztn)$aFK>{4yYO$_1(IJ(e!V~(?`NNRjX2F>-5exO z>t;$Xe2}pFB4JCI(C)c+Vs{GbU>tFnkt(?^;&1n*U$LJp=RO$8{<4dFiWn@t6!Ks{ z_%-6P z|G@rt1^u03|JcQN2c1#gyt_g2+ZmtTYdgr7yBKdb`H;+b4|4u}`)28fGoD8ZMPKxJ zVJ&_?KS%VBIA@+Bub$5re-`;Tm-!pPK9I$Fc!2pRAWu@$r8kCs?PA7l-!CoL8PC0V zKl9%Tf5Up|-%LM?$g?Ka=@HiT7nz@iMUtP#{Ei@=_WkztTh)I2tm10-tr*thR?ZXq z-knX}p23eX+zXDyGp^ObFydB+onI%3zmM_HVSVSazP6!vcbMeXOclPuKCy{;AH;oU zGk)KOAI~uUSJ|hgA^!vB|K<6z7f&ATOA$Sn{$4>omizKS*84-`Lkjy$@_6aJeT^_C zT=*62WU=4=%R=#|ke?ga=j!nP0QLtAlw2G8{*BZRetfC;k05_F=*N#yd`CP>KR;$2 z2Jq(x+0wrQdNlIK$eVKZiy@5rX6FAV#OWo=7yMa(o$uty?itS6chEn?y1E|w2k5UJ zJc!+qe^j1LyGVADkgp|9smM*ZN$rENKX9(-?%Rada;{v1UNd=h5&7CmobF{E-OW0Q zCr_%@%Fa~ccP)4aaZY9Y?-QqqtfP%zkY4g^@Dkwx&Y9=%;}X`j)muu)tIrLV{0Hn$ z*~D!h{cT~NsP;+j3C^prt3|KJ&R5z0Qd7i_Wxnr6FQ2^H1ir^Qx|}?{i*Z@K%DzL} zxc__~dll>#oy=oZvHHutR`?!v-nm$GD{@wsx{Ud?@0s*8$rZA`CtoPKnf2= zy@7l@&N;JasO&D~+*>?RbZw#VA^g3Q`93>D{C7AXl9|76tr34V=Sk!=(W{BmrL4~a z&f@@aESf2~4D5`?PHKPge~R3PoVU5`*RN!#{iF3lt4sbf>tiK(*pG9hjeWJr?hC}_ zYR=JnSx=iL$=+9qR~75P>X0Ad$Ir2Q1@X(6F1^w0FYjJ1`UTe6&K%JKlzhPT&%t{jrkh6TJmeymuE$aE*&qtlKI|G zT;5b%-cco)9RChiSv5qZCsf8kGWFV&N-Y;K0Lzt zJCt=>Ve6ZCTK(4Q+5=f{_p-ivZj;?(6~ zq_|l9y5AttOBmPFtZ%D-@5S!?k&?TId9(Y6)j92*;z9Pq?d*3KkCvX*>z`oXc!_aa z9m(qP_PuL$L#y9RV*j$bg}r+WXI_?!Qa@Hlw)ccO^y0`ftGn(e-u9hr^|yBXw(mzf z&#eAs-;-9)7*4)hJ!$6z`7xdS`2p6m)v*SX=k`uvbvKgqhD#km+iR`zPv2<`jd z>Lrh^7T@X(4b0;y`my&YtKZl@W_440x3KSu3gY-ly@2z00P(VS4g21jevSGcgCD=d ze|t}|dQ8Cp$=UZt7yIlT?E6;VdTfg1#xh>3e>E{4t1C6lkX#(+iQQZ5U95}sVRbP( z*X;Xf6#Kn>xBvSEviD+&a1;GSa1J(eU$MH=4Eo)-RPuLm{@FQbb+N`wwZG1Na473z zE8i>jj$!qtBK(M5F1>}EpBE!a{N{P`jN+yKWL~^URhF_KtSCM0|S>u=-7aefT5J^@q79eUo#gK1_Ps zbA_jfqunz%V)s9phXwhPTZLXa_U(P-H$&BK^?iF+TEKa0@7S+$FZmwvvbxM(>{~tX z7Vh2l?qhYug|X_#-dP_Y|Lh&PmH6BDdsTnQZ^ysoYsNZXYX28Z?bns zt9QPdA-h(u^pi)Aa$bCq_}Tlay;IwM+RVMz&WCT@Dtm3?g?5hFd-h|*-QGK`E@*Wm zdl$BKV0CM|@7wurbx5mg*!!i`X`T(Je|ulHdcVEj+B=@Tho_PU_I+skxb3G_2YZoq zej(?8y*DN@U;Vjn-Ox|{uZ2;b)%{1NwtJo3xxe)hh8?)$vG&)Pex)ibQFY;|4xyAZoy zT3ycGC+ywD>VS6NxA)e>dGgch82wqt_Wo~m)dSoUt!`}V!|FZucMSG-8TNMqR#&#a PH?Vgw`}>X;$*=zc4%cfx literal 0 HcmV?d00001 diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 8459f15..a5ee63d 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -15,7 +15,7 @@ target_compile_options(reports_integration_test target_link_libraries(reports_integration_test PUBLIC sonata_report PRIVATE spdlog::spdlog_header_only - PRIVATE ${MPI_CXX_LIBRARIES} + PRIVATE ${MPI_C_LIBRARIES} ) add_test(NAME reports_integration_test diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 3bc33e1..ab3bc71 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,7 +19,7 @@ target_link_libraries(reports_unit_tests sonata_report Catch2::Catch2 spdlog::spdlog_header_only - ${MPI_CXX_LIBRARIES} + ${MPI_C_LIBRARIES} ${HDF5_C_LIBRARIES} ) diff --git a/tools/converter/CMakeLists.txt b/tools/converter/CMakeLists.txt index e209492..07e59ae 100644 --- a/tools/converter/CMakeLists.txt +++ b/tools/converter/CMakeLists.txt @@ -14,7 +14,7 @@ target_compile_options(spikes_converter target_link_libraries(spikes_converter PUBLIC sonata_report PRIVATE spdlog::spdlog_header_only - PRIVATE ${MPI_CXX_LIBRARIES} + PRIVATE ${MPI_C_LIBRARIES} ) add_executable(reports_converter reports_converter.cpp) @@ -32,7 +32,7 @@ target_link_libraries(reports_converter PUBLIC sonata_report PUBLIC binary_reader PRIVATE spdlog::spdlog_header_only - PRIVATE ${MPI_CXX_LIBRARIES} + PRIVATE ${MPI_C_LIBRARIES} ) install(TARGETS spikes_converter DESTINATION bin) diff --git a/tools/converter/binary_reader/ReadManager.cpp b/tools/converter/binary_reader/ReadManager.cpp index eae6c3f..d44cbb5 100644 --- a/tools/converter/binary_reader/ReadManager.cpp +++ b/tools/converter/binary_reader/ReadManager.cpp @@ -767,11 +767,6 @@ namespace bbpReader { // read. } - void FrameParser::readMultipleFrameData(DataItem* buffer, int num_frames) { - readFrame(buffer); - timestep += num_frames; - } - size_t FrameParser::ReadOptimiser::defaultElementSize = 1; FrameParser::ReadOptimiser::ReadOptimiser() { diff --git a/tools/converter/binary_reader/ReadManager.h b/tools/converter/binary_reader/ReadManager.h index 7b054e3..97abe37 100644 --- a/tools/converter/binary_reader/ReadManager.h +++ b/tools/converter/binary_reader/ReadManager.h @@ -697,7 +697,6 @@ namespace bbpReader { void readFrameMapping(MappingItem* buffer); void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame // after. Data is sorted by GID and segment num - void readMultipleFrameData(DataItem* buffer, int num_frames); /*getReferences() * returns a vector of vectors of offsets that correspond to the relevant segments inside diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index d10aff4..52a2050 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -14,17 +14,6 @@ using namespace bbpReader; -int get_steps_to_write(const char* var_name) { - // In general, writing 10 steps per iteration is best performance wise - // The environment variable STEPS_TO_WRITE could be overwritten for testing - int steps_to_write = 10; - char* var_value = getenv(var_name); - if (var_value != nullptr && atoi(var_value) > 0) { - steps_to_write = atoi(var_value); - } - return steps_to_write; -} - static void show_usage(std::string name) { std::cerr << "Usage: " << name << " \n" << "Options:\n" @@ -65,8 +54,7 @@ int main(int argc, char* argv[]) { FrameParser frame_parser(file_name); Header header = frame_parser.getHeader(); - std::string filename = file_name; - std::string report_name = filename.substr(filename.find_last_of("/\\") + 1); + std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); // Get header information in order to create the report double tstart = header.getStartTime(); @@ -103,27 +91,20 @@ int main(int argc, char* argv[]) { sonata_prepare_datasets(); uint32_t num_steps = header.getNumberOfSteps(); - uint32_t num_frames = get_steps_to_write("STEPS_TO_WRITE"); uint64_t element_ids_per_frame = frame_parser.getBufferSize_elements(); - uint64_t total_element_ids = element_ids_per_frame * num_frames; - logger->info("Number of frames to write per iteration: '{}'", num_frames); - DataItem* all_element_ids_buffer = new DataItem[total_element_ids]; + DataItem* element_ids_buffer = new DataItem[element_ids_per_frame]; uint32_t timestep = 0; // Write the timestep frames while (frame_parser.hasMore()) { - // Check if number of frames to write is not bigger than the remaining - if (num_steps - timestep < num_frames) { - num_frames = num_steps - timestep; - } - frame_parser.readMultipleFrameData(all_element_ids_buffer, num_frames); + frame_parser.readFrameData(element_ids_buffer); sonata_write_buffered_data(report_name.data(), - all_element_ids_buffer, - total_element_ids, - num_frames); + element_ids_buffer, + element_ids_per_frame, + 1); // Number of frames to write if (timestep % 1000 == 0 && timestep > 0) { logger->info("Writing timestep '{}'", timestep); } - timestep += num_frames; + timestep++; } logger->info("'{}' timesteps written!", timestep); logger->info("File '{}' successfully converted to '{}.h5'", file_name, report_name); diff --git a/tools/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp index caf9981..9c4ae34 100644 --- a/tools/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -63,7 +63,8 @@ int main(int argc, char* argv[]) { } // Create a spike file - sonata_create_spikefile(".", file_name.data()); + std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); + sonata_create_spikefile(".", report_name.data()); std::string population_name = "All"; uint64_t population_offset = 0; @@ -77,7 +78,7 @@ int main(int argc, char* argv[]) { // Close the spike file sonata_close_spikefile(); - logger->info("File '{}' successfully converted to '{}.h5'", file_name, file_name); + logger->info("File '{}' successfully converted to '{}.h5'", file_name, report_name); #ifdef SONATA_REPORT_HAVE_MPI MPI_Finalize(); From cfad345afe5ced888e092c270173a6e68a9aaeda Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Fri, 7 Oct 2022 12:37:05 +0200 Subject: [PATCH 07/10] Clang format --- tools/converter/binary_reader/ReadManager.cpp | 1353 ++++++++--------- tools/converter/binary_reader/ReadManager.h | 1186 ++++++++------- .../binary_reader/crossPlatformConversion.h | 8 +- tools/converter/reports_converter.cpp | 1 - 4 files changed, 1275 insertions(+), 1273 deletions(-) diff --git a/tools/converter/binary_reader/ReadManager.cpp b/tools/converter/binary_reader/ReadManager.cpp index d44cbb5..b29aee3 100644 --- a/tools/converter/binary_reader/ReadManager.cpp +++ b/tools/converter/binary_reader/ReadManager.cpp @@ -1,154 +1,143 @@ -#include -#include #include "ReadManager.h" +#include +#include using namespace std; namespace bbpReader { - Header::Header(char* readBuffer) { -#define fetch(TYPE, POSITION) *((TYPE*)(readBuffer + POSITION)) -#define fetchS(POSITION) string((char*)(readBuffer + POSITION)) - - double readIdentifier = fetch(double, IDENTIFIER_POSITION); - headerSize = fetch(int32_t, HEADER_SIZE_POSITION); - amountOfCells = fetch(int32_t, TOTAL_NUMBER_OF_CELLS_POSITION); - /** \bug 32 bit is consistent with the binary format specification of the - header, however - */ - uint32_t compartmentsPerFrame = fetch(int32_t, TOTAL_NUMBER_OF_COMPARTMENTS_POSITION); - libraryVersion = fetchS(LIBRARY_VERSION_POSITION); - simulatorVersion = fetchS(SIMULATOR_VERSION_POSITION); - numberOfSteps = fetch(int32_t, NUMBER_OF_STEPS_POSITION); - startTime = fetch(double, TIME_START_POSITION); - endTime = fetch(double, TIME_END_POSITION); - dt = fetch(double, DT_TIME_POSITION); - dataUnit = fetchS(D_UNIT_POSITION); - timeUnit = fetchS(T_UNIT_POSITION); - mappingSize = fetch(int32_t, MAPPING_SIZE_POSITION); - mappingName = fetchS(MAPPING_NAME_POSITION); - extraMappingSize = fetch(int32_t, EXTRA_MAPPING_SIZE_POSITION); - extraMappingName = fetchS(EXTRA_MAPPING_NAME_POSITION); - targetName = fetchS(TARGET_NAME_POSITION); +Header::Header(char* readBuffer) { +#define fetch(TYPE, POSITION) *((TYPE*) (readBuffer + POSITION)) +#define fetchS(POSITION) string((char*) (readBuffer + POSITION)) + + double readIdentifier = fetch(double, IDENTIFIER_POSITION); + headerSize = fetch(int32_t, HEADER_SIZE_POSITION); + amountOfCells = fetch(int32_t, TOTAL_NUMBER_OF_CELLS_POSITION); + /** \bug 32 bit is consistent with the binary format specification of the + header, however + */ + uint32_t compartmentsPerFrame = fetch(int32_t, TOTAL_NUMBER_OF_COMPARTMENTS_POSITION); + libraryVersion = fetchS(LIBRARY_VERSION_POSITION); + simulatorVersion = fetchS(SIMULATOR_VERSION_POSITION); + numberOfSteps = fetch(int32_t, NUMBER_OF_STEPS_POSITION); + startTime = fetch(double, TIME_START_POSITION); + endTime = fetch(double, TIME_END_POSITION); + dt = fetch(double, DT_TIME_POSITION); + dataUnit = fetchS(D_UNIT_POSITION); + timeUnit = fetchS(T_UNIT_POSITION); + mappingSize = fetch(int32_t, MAPPING_SIZE_POSITION); + mappingName = fetchS(MAPPING_NAME_POSITION); + extraMappingSize = fetch(int32_t, EXTRA_MAPPING_SIZE_POSITION); + extraMappingName = fetchS(EXTRA_MAPPING_NAME_POSITION); + targetName = fetchS(TARGET_NAME_POSITION); #undef fetchS #undef fetch - isNative = readIdentifier == ARCHITECTURE_IDENTIFIER; - - if (!isNative) { - // check if data can be converted: - DOUBLESWAP(readIdentifier); - if (readIdentifier != ARCHITECTURE_IDENTIFIER) { - cerr << "File is corrupt or originated from an unknown architecture." << endl; - exit(1); - } + isNative = readIdentifier == ARCHITECTURE_IDENTIFIER; - // if data was stored on a machine with different architecture, - // then the data needs to be converted to be compatible with the reading machine - FIX_INT(headerSize); - FIX_INT(amountOfCells); - FIX_INT(compartmentsPerFrame); - FIX_INT(numberOfSteps); - DOUBLESWAP(startTime); - DOUBLESWAP(endTime); - DOUBLESWAP(dt); - FIX_INT(mappingSize); - FIX_INT(extraMappingSize); + if (!isNative) { + // check if data can be converted: + DOUBLESWAP(readIdentifier); + if (readIdentifier != ARCHITECTURE_IDENTIFIER) { + cerr << "File is corrupt or originated from an unknown architecture." << endl; + exit(1); } - // printf( "read %d compartment count\n", compartmentsPerFrame ); - totalAmountOfCompartments = compartmentsPerFrame; + // if data was stored on a machine with different architecture, + // then the data needs to be converted to be compatible with the reading machine + FIX_INT(headerSize); + FIX_INT(amountOfCells); + FIX_INT(compartmentsPerFrame); + FIX_INT(numberOfSteps); + DOUBLESWAP(startTime); + DOUBLESWAP(endTime); + DOUBLESWAP(dt); + FIX_INT(mappingSize); + FIX_INT(extraMappingSize); } - bool Header::operator==(const Header& h1) const { + totalAmountOfCompartments = compartmentsPerFrame; +} + +bool Header::operator==(const Header& h1) const { #define _CHECK(field) \ if (h1.field != field) \ return false - // compare all fields other than isNative - _CHECK(headerSize); - _CHECK(libraryVersion); - _CHECK(simulatorVersion); - _CHECK(amountOfCells); - _CHECK(totalAmountOfCompartments); - _CHECK(numberOfSteps); - _CHECK(startTime); - _CHECK(endTime); - _CHECK(dt); - _CHECK(dataUnit); - _CHECK(timeUnit); - _CHECK(mappingSize); - _CHECK(mappingName); - _CHECK(extraMappingName); - _CHECK(extraMappingSize); - _CHECK(targetName); - - // else if nothing returned false: - return true; + // compare all fields other than isNative + _CHECK(headerSize); + _CHECK(libraryVersion); + _CHECK(simulatorVersion); + _CHECK(amountOfCells); + _CHECK(totalAmountOfCompartments); + _CHECK(numberOfSteps); + _CHECK(startTime); + _CHECK(endTime); + _CHECK(dt); + _CHECK(dataUnit); + _CHECK(timeUnit); + _CHECK(mappingSize); + _CHECK(mappingName); + _CHECK(extraMappingName); + _CHECK(extraMappingSize); + _CHECK(targetName); + + // else if nothing returned false: + return true; #undef _CHECK - } +} - CellInfo::CellInfo() { - cellNum = 0; - master = &cellNum; - amountOfCompartments = 0; - dataLocation = 0; - extraMappingLocation = 0; - mappingLocation = 0; - } +CellInfo::CellInfo() { + cellNum = 0; + master = &cellNum; + amountOfCompartments = 0; + dataLocation = 0; + extraMappingLocation = 0; + mappingLocation = 0; +} - CellInfo::CellInfo(char* readBuffer, bool isNative, key cmp) { -#define fetch(TYPE, POSITION) *((TYPE*)(readBuffer + POSITION)) - cellNum = fetch(int32_t, NUMBER_OF_CELL_POSITION); - amountOfCompartments = fetch(int32_t, NUMBER_OF_COMPARTMENTS_POSITION); +CellInfo::CellInfo(char* readBuffer, bool isNative, key cmp) { +#define fetch(TYPE, POSITION) *((TYPE*) (readBuffer + POSITION)) + cellNum = fetch(int32_t, NUMBER_OF_CELL_POSITION); + amountOfCompartments = fetch(int32_t, NUMBER_OF_COMPARTMENTS_POSITION); - dataLocation = fetch(uint64_t, DATA_INFO_POSITION); - extraMappingLocation = fetch(uint64_t, EXTRA_MAPPING_INFO_POSITION); - mappingLocation = fetch(uint64_t, MAPPING_INFO_POSITION); + dataLocation = fetch(uint64_t, DATA_INFO_POSITION); + extraMappingLocation = fetch(uint64_t, EXTRA_MAPPING_INFO_POSITION); + mappingLocation = fetch(uint64_t, MAPPING_INFO_POSITION); #undef fetch - if (!isNative) { - // cout << "Fixing cell info data..." << endl; - FIX_INT(cellNum); - FIX_INT(amountOfCompartments); - - // cout << "size of offset type is " << sizeof(dataLocation) << endl; - DOUBLESWAP(dataLocation); - DOUBLESWAP(extraMappingLocation); - DOUBLESWAP(mappingLocation); - } - - // fprintf( stderr, "build cell info: gid %d dataPos %d extraPos %d mappingPos %d\n", - // cellNum, (int) dataLocation, (int) extraMappingLocation, (int) mappingLocation ); - // cerr<<"build cell info: gid "<) - DEFINE_CMP_OP(>=) +DEFINE_CMP_OP(<) +DEFINE_CMP_OP(<=) +DEFINE_CMP_OP(>) +DEFINE_CMP_OP(>=) #undef DEFINE_CMP_OP #undef COMPARE_ - bool CellInfo::compareByOffset(const CellInfo& a, const CellInfo& b) { - return a.dataLocation < b.dataLocation; - } +bool CellInfo::compareByOffset(const CellInfo& a, const CellInfo& b) { + return a.dataLocation < b.dataLocation; +} - FrameInfo::FrameInfo(char* buffer, char* endOfBuffer, bool isNative, CellInfo::key sortMode) { - // create a cellinfo and then insert into itself (rem, this class inherits from set). The - // resulting set iterator is then put into - // a map member for associative lookup gid:location - for (char* bufferLoc = buffer; bufferLoc < endOfBuffer; - bufferLoc += SIZE_CELL_INFO_LENGTH) { - iterator ins = insert(CellInfo(bufferLoc, isNative, sortMode)).first; - gidLookup.insert(make_pair(ins->getCellNum(), ins)); - } +FrameInfo::FrameInfo(char* buffer, char* endOfBuffer, bool isNative, CellInfo::key sortMode) { + // create a cellinfo and then insert into itself (rem, this class inherits from set). The + // resulting set iterator is then put into + // a map member for associative lookup gid:location + for (char* bufferLoc = buffer; bufferLoc < endOfBuffer; bufferLoc += SIZE_CELL_INFO_LENGTH) { + iterator ins = insert(CellInfo(bufferLoc, isNative, sortMode)).first; + gidLookup.insert(make_pair(ins->getCellNum(), ins)); } +} + +FrameInfo::FrameInfo(const FrameInfo& rhs) + : baseTypes::FrameInfo() { + *this = rhs; +} - FrameInfo::FrameInfo(const FrameInfo& rhs) : baseTypes::FrameInfo() { - *this = rhs; +FrameInfo& FrameInfo::operator=(const FrameInfo& rhs) { + insert(rhs.begin(), rhs.end()); + for (iterator cellItem = begin(); cellItem != end(); ++cellItem) { + gidLookup.insert(make_pair(cellItem->getCellNum(), cellItem)); } - FrameInfo& FrameInfo::operator=(const FrameInfo& rhs) { - insert(rhs.begin(), rhs.end()); - for (iterator cellItem = begin(); cellItem != end(); ++cellItem) { - gidLookup.insert(make_pair(cellItem->getCellNum(), cellItem)); - } + return *this; +} + +FrameInfo& FrameInfo::filterGIDs(const vector& gids) { + Lookup newGidLookup; - return *this; + for (vector::const_iterator gid = gids.begin(); gid != gids.end(); gid++) { + Lookup::iterator it = gidLookup.find(*gid); + if (it != gidLookup.end()) { + newGidLookup.insert(*it); + gidLookup.erase(it); + } else + throw string("GID doesn't exist in file."); // TODO: replace by custom std::exception } - FrameInfo& FrameInfo::filterGIDs(const vector& gids) { - Lookup newGidLookup; - - for (vector::const_iterator gid = gids.begin(); gid != gids.end(); gid++) { - Lookup::iterator it = gidLookup.find(*gid); - if (it != gidLookup.end()) { - newGidLookup.insert(*it); - gidLookup.erase(it); - } else - throw string( - "GID doesn't exist in file."); // TODO: replace by custom std::exception - } + // gidLookup contains the gids that need to be removed and the newGidLookup contains the + // gids that remain. - // gidLookup contains the gids that need to be removed and the newGidLookup contains the - // gids that remain. + for (Lookup::iterator i = gidLookup.begin(); i != gidLookup.end(); i++) + erase(i->second); // delete the unwanted gids. - for (Lookup::iterator i = gidLookup.begin(); i != gidLookup.end(); i++) - erase(i->second); // delete the unwanted gids. + gidLookup.swap(newGidLookup); - gidLookup.swap(newGidLookup); + return *this; +} - return *this; - } +CompartmentMapping::CompartmentMapping() + : Base() {} - CompartmentMapping::CompartmentMapping() : Base() { - } +CompartmentMapping::CompartmentMapping(MappingItem* start, MappingItem* _end) + : Base(start, _end) {} - CompartmentMapping::CompartmentMapping(MappingItem* start, MappingItem* _end) - : Base(start, _end) { - } +CompartmentMapping::CompartmentMapping(MappingItem* buffer, SizeType bufSize) + : Base(buffer, buffer + bufSize) {} - CompartmentMapping::CompartmentMapping(MappingItem* buffer, SizeType bufSize) - : Base(buffer, buffer + bufSize) { - } +SectionData::SectionData() + : Base() {} - SectionData::SectionData() : Base() { - } +SectionData::SectionData(DataItem firstData) { + push_back(firstData); +} - SectionData::SectionData(DataItem firstData) { - push_back(firstData); - } +SectionData::SectionData(DataItem* start, DataItem* _end) + : Base(start, _end) {} - SectionData::SectionData(DataItem* start, DataItem* _end) : Base(start, _end) { - } +bool SectionData::operator==(const SectionData& s1) { + if (s1.size() != size()) + return false; - bool SectionData::operator==(const SectionData& s1) { - if (s1.size() != size()) + const_iterator dat1, dat2; + for (dat1 = s1.begin(), dat2 = begin(); dat2 != end(); dat1++, dat2++) + if (*dat1 != *dat2) return false; - const_iterator dat1, dat2; - for (dat1 = s1.begin(), dat2 = begin(); dat2 != end(); dat1++, dat2++) - if (*dat1 != *dat2) - return false; + return true; +} - return true; - } +bool SectionData::operator!=(const SectionData& s1) { + return !(*this == s1); +} - bool SectionData::operator!=(const SectionData& s1) { - return !(*this == s1); - } +bool SectionData::compareByOriginalLocation(const Identifier& c1, const Identifier& c2) { + return c1.getLocation() < c2.getLocation(); +} - bool SectionData::compareByOriginalLocation(const Identifier& c1, const Identifier& c2) { - return c1.getLocation() < c2.getLocation(); - } +CellData::CellData() { + sectSort = SID; +} - CellData::CellData() { - sectSort = SID; - } +CellData::CellData(SortingKey sorting) { + sectSort = sorting; +} - CellData::CellData(SortingKey sorting) { - sectSort = sorting; - } +CellData::CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { + sectSort = SID; + fillData(dataBuffer, mapping); +} - CellData::CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { - sectSort = SID; - fillData(dataBuffer, mapping); +void CellData::fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { + CompartmentMapping::const_iterator it; + SectionIndex i; + for (it = mapping.begin(), i = 0; it != mapping.end(); it++, dataBuffer++, i++) { + // iteration through all compartments inside cell + DataItem data = *dataBuffer; + + iterator segMatch = find(*it); // find corresponding segment + + if (segMatch == end()) { + // SID was not found. + // this means it wasnt created yet. + // therefore create a new one and insert first data item into it: + iterator ins = + insert(make_pair(SectionData::Identifier(*it, i, sectSort), SectionData(data))) + .first; + // note that it is guaranteed that the above insertion takes place, because this + // block is only entered + // if the sid does not exist inside this CellData object yet. + // if sorting is done by location, this will be unique anyways. + sidLookup.insert(make_pair(*it, ins)); + } else + // SID is already present in CellData. The new data item just needs to be inserted + segMatch->second.push_back(data); } +} - void CellData::fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { - CompartmentMapping::const_iterator it; - SectionIndex i; - for (it = mapping.begin(), i = 0; it != mapping.end(); it++, dataBuffer++, i++) { - // iteration through all compartments inside cell - DataItem data = *dataBuffer; - - iterator segMatch = find(*it); // find corresponding segment - - if (segMatch == end()) { - // SID was not found. - // this means it wasnt created yet. - // therefore create a new one and insert first data item into it: - iterator ins = - insert(make_pair(SectionData::Identifier(*it, i, sectSort), SectionData(data))) - .first; - // note that it is guaranteed that the above insertion takes place, because this - // block is only entered - // if the sid does not exist inside this CellData object yet. - // if sorting is done by location, this will be unique anyways. - sidLookup.insert(make_pair(*it, ins)); - } else - // SID is already present in CellData. The new data item just needs to be inserted - segMatch->second.push_back(data); - } - } +// typedef CellDataImpl UnsortedCellData; - // typedef CellDataImpl UnsortedCellData; +bool CellData::operator==(const CellData& d1) { + if (d1.size() != size()) + return false; - bool CellData::operator==(const CellData& d1) { - if (d1.size() != size()) + const_iterator seg1, seg2; + + for (seg1 = d1.begin(), seg2 = begin(); seg2 != end(); seg1++, seg2++) { + if (seg1->first != seg2->first) + return false; + if (seg1->second != seg2->second) return false; + } - const_iterator seg1, seg2; + return true; +} - for (seg1 = d1.begin(), seg2 = begin(); seg2 != end(); seg1++, seg2++) { - if (seg1->first != seg2->first) - return false; - if (seg1->second != seg2->second) - return false; - } +ReadManager::ReadManager() {} - return true; - } +ReadManager::ReadManager(const string& filePath) + : ifstream(filePath.c_str(), ios::in | ios::binary) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); + // open(filePath); + readHeader(); +} - ReadManager::ReadManager() { - } +void ReadManager::open(const string& filePath) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); - ReadManager::ReadManager(const string& filePath) - : ifstream(filePath.c_str(), ios::in | ios::binary) { - exceptions(ios::badbit | ios::eofbit | ios::failbit); - // open(filePath); - readHeader(); - } + ifstream::open(filePath.c_str(), ios::in | ios::binary); + readHeader(); +} - void ReadManager::open(const string& filePath) { - exceptions(ios::badbit | ios::eofbit | ios::failbit); +void ReadManager::readHeader() { + char readBuffer[HEADER_LENGTH + 1]; - ifstream::open(filePath.c_str(), ios::in | ios::binary); - readHeader(); - } + seekg(ios::beg); - void ReadManager::readHeader() { - char readBuffer[HEADER_LENGTH + 1]; + ifstream::read(readBuffer, HEADER_LENGTH); - seekg(ios::beg); + readBuffer[HEADER_LENGTH] = 0; // c string terminator + fileInfo = readBuffer; - ifstream::read(readBuffer, HEADER_LENGTH); + cellInfoOrigin = tellg(); - readBuffer[HEADER_LENGTH] = 0; // c string terminator - fileInfo = readBuffer; + isNative = fileInfo.sourceIsNative(); - cellInfoOrigin = tellg(); + CellInfo firstCell = retrieveCellInfo(0); - isNative = fileInfo.sourceIsNative(); + startLocation = firstCell.getDataLocation(); + mappingLocation = firstCell.getMappingLocation(); - CellInfo firstCell = retrieveCellInfo(0); + mappingSize_floats = fileInfo.getMappingSize() * fileInfo.getTotalNumberOfCompartments(); +} - startLocation = firstCell.getDataLocation(); - mappingLocation = firstCell.getMappingLocation(); +CellInfo ReadManager::makeNullInfo() { + char fakeBuffer[SIZE_CELL_INFO_LENGTH]; - mappingSize_floats = fileInfo.getMappingSize() * fileInfo.getTotalNumberOfCompartments(); - } + for (char *i = fakeBuffer, *bufferEnd = fakeBuffer + SIZE_CELL_INFO_LENGTH; i < bufferEnd; i++) + *i = 0; - CellInfo ReadManager::makeNullInfo() { - char fakeBuffer[SIZE_CELL_INFO_LENGTH]; + return CellInfo(fakeBuffer); +} - for (char *i = fakeBuffer, *bufferEnd = fakeBuffer + SIZE_CELL_INFO_LENGTH; i < bufferEnd; - i++) - *i = 0; +const CellInfo ReadManager::_null_info = ReadManager::makeNullInfo(); - return CellInfo(fakeBuffer); - } +CellInfo ReadManager::retrieveFindCell(CellID cellNum) { + int start = (cellNum > fileInfo.getNumberOfCells()) + ? fileInfo.getNumberOfCells() - 1 + : cellNum - 1; // in many cases cellNum will be same as cellIndex + CellInfo seek; + + bool oppositeSearch = false; - const CellInfo ReadManager::_null_info = ReadManager::makeNullInfo(); - - CellInfo ReadManager::retrieveFindCell(CellID cellNum) { - int start = (cellNum > fileInfo.getNumberOfCells()) - ? fileInfo.getNumberOfCells() - 1 - : cellNum - 1; // in many cases cellNum will be same as cellIndex - CellInfo seek; - - bool oppositeSearch = false; - - int i = start; - do { - if (i >= (int)fileInfo.getNumberOfCells() || i < 0) { - if (oppositeSearch) - return makeNullInfo(); - else { - oppositeSearch = true; - i = start; - } + int i = start; + do { + if (i >= (int) fileInfo.getNumberOfCells() || i < 0) { + if (oppositeSearch) + return makeNullInfo(); + else { + oppositeSearch = true; + i = start; } + } - seek = retrieveCellInfo(i); + seek = retrieveCellInfo(i); - if ((int)seek.getCellNum() < i) - if (oppositeSearch) - i--; - else - i++; - else if (oppositeSearch) - i++; - else + if ((int) seek.getCellNum() < i) + if (oppositeSearch) i--; + else + i++; + else if (oppositeSearch) + i++; + else + i--; - } while (seek.getCellNum() != cellNum); + } while (seek.getCellNum() != cellNum); - return seek; - } + return seek; +} - CellInfo ReadManager::retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode) { - FilePosition ppos = tellg(); +CellInfo ReadManager::retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode) { + FilePosition ppos = tellg(); - seekgCellInfo(cellIndex); + seekgCellInfo(cellIndex); - char readBuffer[SIZE_CELL_INFO_LENGTH]; - read(readBuffer, SIZE_CELL_INFO_LENGTH); + char readBuffer[SIZE_CELL_INFO_LENGTH]; + read(readBuffer, SIZE_CELL_INFO_LENGTH); - CellInfo reqInfo(readBuffer, isNative, sortMode); + CellInfo reqInfo(readBuffer, isNative, sortMode); - seekg(ppos); + seekg(ppos); - return reqInfo; - } + return reqInfo; +} - FrameInfo ReadManager::retrieveAllCellInfo(CellInfo::key sortMode) { - FilePosition ppos = tellg(); +FrameInfo ReadManager::retrieveAllCellInfo(CellInfo::key sortMode) { + FilePosition ppos = tellg(); - CellCount amountOfCells = fileInfo.getNumberOfCells(); - streamsize bytesToRead = SIZE_CELL_INFO_LENGTH * amountOfCells; - char* readBuffer = new char[bytesToRead]; - char* endOfBuffer = readBuffer + bytesToRead; - seekgCellInfo(0); + CellCount amountOfCells = fileInfo.getNumberOfCells(); + streamsize bytesToRead = SIZE_CELL_INFO_LENGTH * amountOfCells; + char* readBuffer = new char[bytesToRead]; + char* endOfBuffer = readBuffer + bytesToRead; + seekgCellInfo(0); - read(readBuffer, bytesToRead); + read(readBuffer, bytesToRead); - FrameInfo allCells(readBuffer, endOfBuffer, isNative, sortMode); + FrameInfo allCells(readBuffer, endOfBuffer, isNative, sortMode); - delete[] readBuffer; + delete[] readBuffer; - seekg(ppos); + seekg(ppos); - return allCells; - } + return allCells; +} - void ReadManager::readFrame(DataItem* dataBuffer) { - read(dataBuffer, fileInfo.getTotalNumberOfCompartments()); - } +void ReadManager::readFrame(DataItem* dataBuffer) { + read(dataBuffer, fileInfo.getTotalNumberOfCompartments()); +} - void ReadManager::readFrameMapping(MappingItem* buffer) { - FilePosition ppos = tellg(); +void ReadManager::readFrameMapping(MappingItem* buffer) { + FilePosition ppos = tellg(); - seekgMappingStart(); + seekgMappingStart(); - read(buffer, mappingSize_floats); + read(buffer, mappingSize_floats); - seekg(ppos); - } + seekg(ppos); +} - FrameParser::FrameOrganiser::FrameOrganiser() { - sectSort = SID; - } +FrameParser::FrameOrganiser::FrameOrganiser() { + sectSort = SID; +} - FrameParser::FrameOrganiser::FrameOrganiser(SortingKey segmentSorting) { - sectSort = segmentSorting; - } +FrameParser::FrameOrganiser::FrameOrganiser(SortingKey segmentSorting) { + sectSort = segmentSorting; +} - void FrameParser::FrameOrganiser::fillData(const FrameInfo& info, - const vector& mapping, - const DataItem* dataBuffer) { - FrameInfo::const_iterator pinf = info.begin(); - vector::const_iterator pmap = mapping.begin(); +void FrameParser::FrameOrganiser::fillData(const FrameInfo& info, + const vector& mapping, + const DataItem* dataBuffer) { + FrameInfo::const_iterator pinf = info.begin(); + vector::const_iterator pmap = mapping.begin(); - for (const DataItem *ptr = dataBuffer; pinf != info.end(); - ptr += pinf->getAmountOfCompartments(), pinf++, pmap++) { - // iteration through all gids + for (const DataItem* ptr = dataBuffer; pinf != info.end(); + ptr += pinf->getAmountOfCompartments(), pinf++, pmap++) { + // iteration through all gids - iterator ins = insert(make_pair(*pinf, CellData(sectSort))) - .first; // insert CellData object for current gid - ins->second.fillData(ptr, *pmap); // feed new CellData object with mapping and data - } + iterator ins = insert(make_pair(*pinf, CellData(sectSort))).first; // insert CellData + // object for current + // gid + ins->second.fillData(ptr, *pmap); // feed new CellData object with mapping and data } +} - void FrameParser::FrameOrganiser::makeOrderedDataBuffer(DataItem* buffer) const { - for (const_iterator cd = begin(); cd != end(); cd++) // iterate through cells - for (CellData::const_iterator sd = cd->second.begin(); sd != cd->second.end(); - sd++) // iterate through segments - for (SectionData::const_iterator f = sd->second.begin(); f != sd->second.end(); - f++, buffer++) // iterate through compartments - *buffer = *f; - } +void FrameParser::FrameOrganiser::makeOrderedDataBuffer(DataItem* buffer) const { + for (const_iterator cd = begin(); cd != end(); cd++) // iterate through cells + for (CellData::const_iterator sd = cd->second.begin(); sd != cd->second.end(); + sd++) // iterate through segments + for (SectionData::const_iterator f = sd->second.begin(); f != sd->second.end(); + f++, buffer++) // iterate through compartments + *buffer = *f; +} - FrameParser::CompartmentIndexing FrameParser::FrameOrganiser::makeOrderedOffsetReferences() - const { - CompartmentIndexing gidV; - gidV.reserve(size()); +FrameParser::CompartmentIndexing FrameParser::FrameOrganiser::makeOrderedOffsetReferences() const { + CompartmentIndexing gidV; + gidV.reserve(size()); - FrameDataIndex position = 0; + FrameDataIndex position = 0; - for (const_iterator gid = begin(); gid != end(); gid++) { // iterate through gids - gidV.push_back(vector()); + for (const_iterator gid = begin(); gid != end(); gid++) { // iterate through gids + gidV.push_back(vector()); - // reserve memory for amount of segments - gidV.back().reserve((int)(gid->second.size() * 1.5)); + // reserve memory for amount of segments + gidV.back().reserve((int) (gid->second.size() * 1.5)); - MappingItem presid = 0; - for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); - position += sid->second.size(), sid++) { - // iterating through available sid's - while (presid++ != sid->first.getSID()) - gidV.back().push_back(0); // insert dummy variables for unavailable sids (gaps) - gidV.back().push_back(position); - } + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); + position += sid->second.size(), sid++) { + // iterating through available sid's + while (presid++ != sid->first.getSID()) + gidV.back().push_back(0); // insert dummy variables for unavailable sids (gaps) + gidV.back().push_back(position); } - - return gidV; } - FrameParser::CompartmentCounts FrameParser::FrameOrganiser::makeOrderedSegSizes() const { - CompartmentCounts gidV; - gidV.reserve(size()); - - for (const_iterator gid = begin(); gid != end(); gid++) { - gidV.push_back(vector()); - gidV.back().reserve((int)(gid->second.size() * 1.5)); - MappingItem presid = 0; - for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); - sid++) { - while (presid++ != sid->first.getSID()) - gidV.back().push_back(0); // fill sid gaps with zero sizes - gidV.back().push_back(sid->second.size()); - } - } + return gidV; +} - return gidV; +FrameParser::CompartmentCounts FrameParser::FrameOrganiser::makeOrderedSegSizes() const { + CompartmentCounts gidV; + gidV.reserve(size()); + + for (const_iterator gid = begin(); gid != end(); gid++) { + gidV.push_back(vector()); + gidV.back().reserve((int) (gid->second.size() * 1.5)); + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); sid++) { + while (presid++ != sid->first.getSID()) + gidV.back().push_back(0); // fill sid gaps with zero sizes + gidV.back().push_back(sid->second.size()); + } } - FrameParser::FrameParser() { - readBuffer = NULL; - refArray = NULL; + return gidV; +} + +FrameParser::FrameParser() { + readBuffer = NULL; + refArray = NULL; +} + +FrameParser::FrameParser(const string& file, bool sortData) + : ReadManager(file) { + if (fileInfo.getMappingSize() != 1) { + cerr << "Target file has incompatible mapping size for FrameParser. FrameParser only works " + "with mapping size 1." + << endl; + exit(1); } - FrameParser::FrameParser(const string& file, bool sortData) : ReadManager(file) { - if (fileInfo.getMappingSize() != 1) { - cerr - << "Target file has incompatible mapping size for FrameParser. FrameParser only works with mapping size 1." - << endl; - exit(1); - } + CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; - CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; + FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); - FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); + ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); + offsets = ReadOptimiser(fileInfo.getTotalNumberOfCompartments()); // * sizeof(DataItem) ); + createReferences(cinfo, sortData); - ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); - offsets = ReadOptimiser(fileInfo.getTotalNumberOfCompartments()); // * sizeof(DataItem) ); - createReferences(cinfo, sortData); + timestep = 0; // by default start pointing to timstep 0 + ReadManager::seekgFrame(timestep); +} - timestep = 0; // by default start pointing to timstep 0 - ReadManager::seekgFrame(timestep); - } +FrameParser::FrameParser(const string& file, const vector& gidTarget, bool sortData) + : ReadManager(file) { + timestep = 0; + ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); + retarget(gidTarget, sortData); +} - FrameParser::FrameParser(const string& file, const vector& gidTarget, bool sortData) - : ReadManager(file) { - timestep = 0; - ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); - retarget(gidTarget, sortData); - } +FrameParser::~FrameParser() { + if (readBuffer) // readBuffer may have not been used if sorting was disabled, in which case + // it would have been set to NULL + delete[] readBuffer; + if (refArray) // same applies for refArray + delete[] refArray; +} - FrameParser::~FrameParser() { - if (readBuffer) // readBuffer may have not been used if sorting was disabled, in which case - // it would have been set to NULL - delete[] readBuffer; - if (refArray) // same applies for refArray - delete[] refArray; - } +void FrameParser::retarget(const vector& gidTarget, bool sortData) { + CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; - void FrameParser::retarget(const vector& gidTarget, bool sortData) { - CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; + FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); + cinfo.filterGIDs(gidTarget); - FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); - cinfo.filterGIDs(gidTarget); + offsets = ReadOptimiser(cinfo, getFrameSize_bytes(), getStartOffset()); + offsets.setElementSize(sizeof(DataItem)); - offsets = ReadOptimiser(cinfo, getFrameSize_bytes(), getStartOffset()); - offsets.setElementSize(sizeof(DataItem)); + createReferences(cinfo, sortData); - createReferences(cinfo, sortData); + seekg(getStartOffset() + offsets.getFirst()); - seekg(getStartOffset() + offsets.getFirst()); + operator=(timestep); +} - operator=(timestep); - } +void FrameParser::createReferences(const FrameInfo& cinfo, bool sortData) { + SortingKey sSorting = (sortData) ? SID : LOC; - void FrameParser::createReferences(const FrameInfo& cinfo, bool sortData) { - SortingKey sSorting = (sortData) ? SID : LOC; + refArray = NULL; + readBuffer = NULL; - refArray = NULL; - readBuffer = NULL; + vector allMapping; - vector allMapping; + DataItem* tempBuffer = + new DataItem[offsets.getDataSize_elements()]; // allocate memory for readBuffer. This + // will atleast be needed for creating + // the FrameOrganiser - DataItem* tempBuffer = - new DataItem[offsets.getDataSize_elements()]; // allocate memory for readBuffer. This - // will atleast be needed for creating - // the FrameOrganiser + readFrameMapping(tempBuffer); - readFrameMapping(tempBuffer); + FrameInfo::const_iterator pinf = cinfo.begin(); - FrameInfo::const_iterator pinf = cinfo.begin(); + for (DataItem *mpptr = tempBuffer, *endptr = mpptr + pinf->getAmountOfCompartments(); + pinf != cinfo.end(); + pinf++, mpptr = endptr, endptr += pinf->getAmountOfCompartments()) + allMapping.push_back(CompartmentMapping(mpptr, endptr)); - for (DataItem *mpptr = tempBuffer, *endptr = mpptr + pinf->getAmountOfCompartments(); - pinf != cinfo.end(); pinf++, mpptr = endptr, endptr += pinf->getAmountOfCompartments()) - allMapping.push_back(CompartmentMapping(mpptr, endptr)); + if (sortData) + for (FrameDataIndex i = 0; i < offsets.getDataSize_elements(); i++) + tempBuffer[i] = i; // makes buffer of indexes { 0 , 1 ,2 , 3 , ... } + // else it doesnt really matter what FrameOrganiser is fed with. - if (sortData) - for (FrameDataIndex i = 0; i < offsets.getDataSize_elements(); i++) - tempBuffer[i] = i; // makes buffer of indexes { 0 , 1 ,2 , 3 , ... } - // else it doesnt really matter what FrameOrganiser is fed with. + org = FrameOrganiser(sSorting); - org = FrameOrganiser(sSorting); + org.fillData(cinfo, allMapping, tempBuffer); // feed the FrameOrganiser with the data. - org.fillData(cinfo, allMapping, tempBuffer); // feed the FrameOrganiser with the data. + // references = org.makeOrderedOffsetReferences(); + // segSizes = org.makeOrderedSegSizes(); - // references = org.makeOrderedOffsetReferences(); - // segSizes = org.makeOrderedSegSizes(); + if (sortData) { + readBuffer = tempBuffer; // keep allocated memory at tempBuffer - if (sortData) { - readBuffer = tempBuffer; // keep allocated memory at tempBuffer + // sortData is enabled -> create an array of pointers to readBuffer + // in which the pointers are sorted by GID and SID of the data the point to. + // these pointers are to be stored into ref array + org.makeOrderedDataBuffer(readBuffer); // rearranges indexes to correspond to gid and + // segment order + // the memory allocated for readBuffer is now being used for a different purpose: + // it contains the sorted indexes, which now need to be converted to pointers to + // readBuffer. - // sortData is enabled -> create an array of pointers to readBuffer - // in which the pointers are sorted by GID and SID of the data the point to. - // these pointers are to be stored into ref array - org.makeOrderedDataBuffer( - readBuffer); // rearranges indexes to correspond to gid and segment order - // the memory allocated for readBuffer is now being used for a different purpose: - // it contains the sorted indexes, which now need to be converted to pointers to - // readBuffer. + refArray = new DataItem*[offsets.getDataSize_elements()]; // allocate memory for + // refArray (will contain + // pointers for sorting) - refArray = new DataItem*[offsets.getDataSize_elements()]; // allocate memory for - // refArray (will contain - // pointers for sorting) + for (float *pIndex = readBuffer, + **pPtr = refArray, + *pEnd = readBuffer + offsets.getDataSize_elements(); + pIndex != pEnd; + pIndex++, pPtr++) + *pPtr = &readBuffer[(FrameDataIndex) *pIndex]; // converts indexes to pointers and + // stores them into refArray + } else + // data sorting is disabled -> allocated memory at temp Buffer is no longer needed. + //(data will be read directly into the memory that user has provided) + delete[] tempBuffer; +} - for (float *pIndex = readBuffer, **pPtr = refArray, - *pEnd = readBuffer + offsets.getDataSize_elements(); - pIndex != pEnd; pIndex++, pPtr++) - *pPtr = &readBuffer[(FrameDataIndex)*pIndex]; // converts indexes to pointers and - // stores them into refArray - } else - // data sorting is disabled -> allocated memory at temp Buffer is no longer needed. - //(data will be read directly into the memory that user has provided) - delete[] tempBuffer; - } +void FrameParser::bufferTransfer(DataItem* buffer) const { + DataItem* reqLocation = buffer; - void FrameParser::bufferTransfer(DataItem* buffer) const { - DataItem* reqLocation = buffer; + if (buffer == readBuffer) // source and destination is the same - trouble! + buffer = new DataItem[offsets.getDataSize_elements()]; // need to allocate different + // destination - if (buffer == readBuffer) // source and destination is the same - trouble! - buffer = new DataItem[offsets.getDataSize_elements()]; // need to allocate different - // destination + for (DataItem **sourcePtr = refArray, + *target = buffer, + *dEnd = buffer + offsets.getDataSize_elements(); + target != dEnd; + sourcePtr++, target++) + *target = **sourcePtr; - for (DataItem **sourcePtr = refArray, *target = buffer, + if (reqLocation != buffer) { // buffer has not been transfered where requested! + // this can now be corrected. + for (DataItem *origin = buffer, + *destination = reqLocation, *dEnd = buffer + offsets.getDataSize_elements(); - target != dEnd; sourcePtr++, target++) - *target = **sourcePtr; - - if (reqLocation != buffer) { // buffer has not been transfered where requested! - // this can now be corrected. - for (DataItem *origin = buffer, *destination = reqLocation, - *dEnd = buffer + offsets.getDataSize_elements(); - origin != dEnd; origin++, destination++) - *destination = *origin; // copy data to requested location - delete[] buffer; // buffer was is only a temporary memory allocation - delete it. - } - } - - FrameParser& FrameParser::operator++() { - timestep++; - seekg(fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; + origin != dEnd; + origin++, destination++) + *destination = *origin; // copy data to requested location + delete[] buffer; // buffer was is only a temporary memory allocation - delete it. } +} - FrameParser& FrameParser::operator++(int) { - return this->operator++(); - } +FrameParser& FrameParser::operator++() { + timestep++; + seekg(fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; +} - FrameParser& FrameParser::operator--() { - timestep--; - seekg(-fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; - } +FrameParser& FrameParser::operator++(int) { + return this->operator++(); +} - FrameParser& FrameParser::operator--(int) { - return this->operator--(); - } +FrameParser& FrameParser::operator--() { + timestep--; + seekg(-fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; +} - FrameParser& FrameParser::operator+=(FrameDiff increment) { - timestep += increment; - seekg(increment * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; - } +FrameParser& FrameParser::operator--(int) { + return this->operator--(); +} - FrameParser& FrameParser::operator-=(FrameDiff decrement) { - timestep -= decrement; - seekg(-decrement * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; - } +FrameParser& FrameParser::operator+=(FrameDiff increment) { + timestep += increment; + seekg(increment * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; +} - FrameParser& FrameParser::operator=(FrameIndex newTime) { - timestep = newTime; - seekgFrame(timestep); - seekg(offsets.getFirst(), cur); - return *this; - } +FrameParser& FrameParser::operator-=(FrameDiff decrement) { + timestep -= decrement; + seekg(-decrement * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); + return *this; +} - Time FrameParser::getTime() const { - return fileInfo.getStartTime() + timestep * fileInfo.getTimeStepSize(); - } +FrameParser& FrameParser::operator=(FrameIndex newTime) { + timestep = newTime; + seekgFrame(timestep); + seekg(offsets.getFirst(), cur); + return *this; +} - void FrameParser::readFrame(DataItem* buffer) { - DataItem* readPtr; +Time FrameParser::getTime() const { + return fileInfo.getStartTime() + timestep * fileInfo.getTimeStepSize(); +} - if (readBuffer) // readBuffer will be NULL if sorting is disabled - readPtr = readBuffer; // sorting enabled -> read into readBuffer and transfer data in - // order later - else - readPtr = buffer; // sorting disabled -> read directly into buffer +void FrameParser::readFrame(DataItem* buffer) { + DataItem* readPtr; - // offsets contains alternating values of data sizes to be read and skipped. - for (ReadOptimiser::iterator i = offsets.begin(); i != offsets.end(); i++) { - read(readPtr, *i); - readPtr += *i; - i++; + if (readBuffer) // readBuffer will be NULL if sorting is disabled + readPtr = readBuffer; // sorting enabled -> read into readBuffer and transfer data in + // order later + else + readPtr = buffer; // sorting disabled -> read directly into buffer - // seems to be confusion as to whether offsets should contain bytelength or element - // count. I have it contain element count for now - // i now points to an offset that indicates amount of data to be skipped - if (i == offsets.end()) // no more offsets -> i'm done reading - break; + // offsets contains alternating values of data sizes to be read and skipped. + for (ReadOptimiser::iterator i = offsets.begin(); i != offsets.end(); i++) { + read(readPtr, *i); + readPtr += *i; + i++; - if (*i) - seekg(*i, ios::cur); // non-zero offset -> perfom the skip - } + // seems to be confusion as to whether offsets should contain bytelength or element + // count. I have it contain element count for now + // i now points to an offset that indicates amount of data to be skipped + if (i == offsets.end()) // no more offsets -> i'm done reading + break; - if (refArray) // refArray will be NULL if sorting is disabled - bufferTransfer(buffer); // sorting is enabled. Transfer data from readBuffer to buffer - // such that data is in order + if (*i) + seekg(*i, ios::cur); // non-zero offset -> perfom the skip } - void FrameParser::readFrameMapping(MappingItem* buffer) { - // readFrameMapping should not return without the filepointer being in the same state as - // before it was called. - FilePosition before = tellg(); // make a backup for current filepointer - - seekgMappingStart(); // now move file pointer to mapping location + if (refArray) // refArray will be NULL if sorting is disabled + bufferTransfer(buffer); // sorting is enabled. Transfer data from readBuffer to buffer + // such that data is in order +} - size_t prevElement = offsets.getElementSize(); // backup element size. - offsets.setElementSize(sizeof(MappingItem)); // setup offsets to be reading MappingItems - // instead of whatever it was reading before. +void FrameParser::readFrameMapping(MappingItem* buffer) { + // readFrameMapping should not return without the filepointer being in the same state as + // before it was called. + FilePosition before = tellg(); // make a backup for current filepointer - seekg(offsets.getFirst(), ios::cur); // and advance to first cell - readFrame( - buffer); // read the data - this function will also alter the filepointer and timestep + seekgMappingStart(); // now move file pointer to mapping location - offsets.setElementSize(prevElement); // restore element size. + size_t prevElement = offsets.getElementSize(); // backup element size. + offsets.setElementSize(sizeof(MappingItem)); // setup offsets to be reading MappingItems + // instead of whatever it was reading before. - seekg(before); // restore filepointer - } + seekg(offsets.getFirst(), ios::cur); // and advance to first cell + readFrame(buffer); // read the data - this function will also alter the filepointer and + // timestep - void FrameParser::readFrameData(DataItem* buffer) { - readFrame(buffer); - timestep++; // the filepointer has been incremented to the next timestep as data has been - // read. - } + offsets.setElementSize(prevElement); // restore element size. - size_t FrameParser::ReadOptimiser::defaultElementSize = 1; - - FrameParser::ReadOptimiser::ReadOptimiser() { - elementSize = 0; - firstOffset = 0; - dataSize_elements = 0; - } + seekg(before); // restore filepointer +} - FrameParser::ReadOptimiser::ReadOptimiser(std::streamsize frameSize_bytes) { - elementSize = defaultElementSize; - firstOffset = 0; - dataSize_elements = frameSize_bytes; // * elementSize; - push_back(dataSize_elements); - } +void FrameParser::readFrameData(DataItem* buffer) { + readFrame(buffer); + timestep++; // the filepointer has been incremented to the next timestep as data has been + // read. +} - FrameParser::ReadOptimiser::ReadOptimiser(const FrameInfo& cellsToRead, - std::streamsize frameSize_bytes, - FilePosition startLocation) { - elementSize = defaultElementSize; - // collect data edge points - multiset dataEdges; +size_t FrameParser::ReadOptimiser::defaultElementSize = 1; - dataEdges.insert(startLocation); +FrameParser::ReadOptimiser::ReadOptimiser() { + elementSize = 0; + firstOffset = 0; + dataSize_elements = 0; +} - for (FrameInfo::const_iterator i = cellsToRead.begin(); i != cellsToRead.end(); i++) { - dataEdges.insert(i->getDataLocation()); // starting point - dataEdges.insert(i->getDataLocation() + - i->getAmountOfCompartments() * elementSize); // end point - } +FrameParser::ReadOptimiser::ReadOptimiser(std::streamsize frameSize_bytes) { + elementSize = defaultElementSize; + firstOffset = 0; + dataSize_elements = frameSize_bytes; // * elementSize; + push_back(dataSize_elements); +} - dataEdges.insert(startLocation + (FileOffset)frameSize_bytes); - - /*create a vector that contains the offsets. - * The first offset represents the first bunch of data to be read. - * The second offset then represents the amound of data to be skipped after the first read. - * and then the next offset is again the amount of data to be read. - */ - - // compute the offsets differentiating the dataEdges - list offsets; - for (multiset::iterator currentPos = dataEdges.begin(), - prevPos = currentPos++; - currentPos != dataEdges.end(); prevPos = currentPos, currentPos++) - offsets.push_back(*currentPos - *prevPos); - - // the last offsets should move the file pointer beyond the frame to the start location on - // the next frame - // therefore the first and last offsets are special -> take them out - - firstOffset = offsets.front(); - FileOffset lastOffset = offsets.back(); - - offsets.pop_front(); // remove first offset from list - offsets.pop_back(); // remove last offset from list - - // now the first an last offset are read dimensions. This guarantees that a skip has always - // a precessor and successor - // this is convenient in the next step, as a skip is always on an even location aswell as - // offsets.end() - - list::iterator it; - bool toggleOnRead; - - dataSize_elements = 0; - for (toggleOnRead = true, it = offsets.begin(); it != offsets.end(); - toggleOnRead = !toggleOnRead, it++) { - if (toggleOnRead) { - dataSize_elements += (*it /= elementSize); - // handle read - // read specifications should be in units of floats, but skips should be in units of - // bytes. - // also size of data to be read per frame needs to established - } else - // handle skip - There should be no zero offset inbetween - if (*it == 0) { - // remove this zero skip by combining prev. and next read. - list::iterator prevRead = it; - list::iterator nextRead = it; - prevRead--; - nextRead++; - - dataSize_elements += - (*nextRead /= elementSize); // next read was still in units of bytes - *prevRead += *nextRead; // combine nextRead with prevRead - offsets.erase(it); - offsets.erase(nextRead); - it = prevRead; // point to prevRead now - toggleOnRead = - true; // and mark that toggle flag that current iteration deal with read. - } +FrameParser::ReadOptimiser::ReadOptimiser(const FrameInfo& cellsToRead, + std::streamsize frameSize_bytes, + FilePosition startLocation) { + elementSize = defaultElementSize; + // collect data edge points + multiset dataEdges; + + dataEdges.insert(startLocation); + + for (FrameInfo::const_iterator i = cellsToRead.begin(); i != cellsToRead.end(); i++) { + dataEdges.insert(i->getDataLocation()); // starting point + dataEdges.insert(i->getDataLocation() + + i->getAmountOfCompartments() * elementSize); // end point + } + + dataEdges.insert(startLocation + (FileOffset) frameSize_bytes); + + /*create a vector that contains the offsets. + * The first offset represents the first bunch of data to be read. + * The second offset then represents the amound of data to be skipped after the first read. + * and then the next offset is again the amount of data to be read. + */ + + // compute the offsets differentiating the dataEdges + list offsets; + for (multiset::iterator currentPos = dataEdges.begin(), prevPos = currentPos++; + currentPos != dataEdges.end(); + prevPos = currentPos, currentPos++) + offsets.push_back(*currentPos - *prevPos); + + // the last offsets should move the file pointer beyond the frame to the start location on + // the next frame + // therefore the first and last offsets are special -> take them out + + firstOffset = offsets.front(); + FileOffset lastOffset = offsets.back(); + + offsets.pop_front(); // remove first offset from list + offsets.pop_back(); // remove last offset from list + + // now the first an last offset are read dimensions. This guarantees that a skip has always + // a precessor and successor + // this is convenient in the next step, as a skip is always on an even location aswell as + // offsets.end() + + list::iterator it; + bool toggleOnRead; + + dataSize_elements = 0; + for (toggleOnRead = true, it = offsets.begin(); it != offsets.end(); + toggleOnRead = !toggleOnRead, it++) { + if (toggleOnRead) { + dataSize_elements += (*it /= elementSize); + // handle read + // read specifications should be in units of floats, but skips should be in units of + // bytes. + // also size of data to be read per frame needs to established + } else + // handle skip - There should be no zero offset inbetween + if (*it == 0) { + // remove this zero skip by combining prev. and next read. + list::iterator prevRead = it; + list::iterator nextRead = it; + prevRead--; + nextRead++; + + dataSize_elements += (*nextRead /= elementSize); // next read was still in units of + // bytes + *prevRead += *nextRead; // combine nextRead with prevRead + offsets.erase(it); + offsets.erase(nextRead); + it = prevRead; // point to prevRead now + toggleOnRead = true; // and mark that toggle flag that current iteration deal with + // read. } + } - // now optimise access performance by converting the offsets list to a vector. + // now optimise access performance by converting the offsets list to a vector. - vector voffsets(offsets.begin(), offsets.end()); - swap(voffsets); + vector voffsets(offsets.begin(), offsets.end()); + swap(voffsets); - // compute the finalSkip, which brings the filePointer back to the starting location, but of - // the nextFrame - FileOffset finalSkip = lastOffset + firstOffset; + // compute the finalSkip, which brings the filePointer back to the starting location, but of + // the nextFrame + FileOffset finalSkip = lastOffset + firstOffset; - if (finalSkip) - push_back(finalSkip); - } + if (finalSkip) + push_back(finalSkip); +} - void FrameParser::ReadOptimiser::updateOffsets(size_t newSize) { - dataSize_elements = newSize * dataSize_elements / elementSize; - for (iterator i = begin(); i != end(); i++) { // iterate through all read offsets - // first reduce the read offsets to byte numbers, then convert it to the new element - // size - *i = newSize * (*i) / elementSize; - i++; - if (i == end()) - break; - } +void FrameParser::ReadOptimiser::updateOffsets(size_t newSize) { + dataSize_elements = newSize * dataSize_elements / elementSize; + for (iterator i = begin(); i != end(); i++) { // iterate through all read offsets + // first reduce the read offsets to byte numbers, then convert it to the new element + // size + *i = newSize * (*i) / elementSize; + i++; + if (i == end()) + break; } } +} // namespace bbpReader diff --git a/tools/converter/binary_reader/ReadManager.h b/tools/converter/binary_reader/ReadManager.h index 97abe37..76b86ea 100644 --- a/tools/converter/binary_reader/ReadManager.h +++ b/tools/converter/binary_reader/ReadManager.h @@ -1,17 +1,18 @@ #ifndef _ReadManager__ #define _ReadManager__ +/** For reference: git@bbpgitlab.epfl.ch:hpc/reportinglib.git */ #include #include #include -#include +#include +#include #include #include -#include -#include #include +#include #include "BinaryPositions.h" @@ -28,695 +29,706 @@ namespace bbpReader { - // class list: - class Header; - class CellInfo; - class FrameInfo; - class CompartmentMapping; - class SectionData; - // class SectionData::Identifier; - class Identifier; - class CellData; - class ReadManager; - class FrameParser; - // class FrameParser::FrameOrganiser; - // class FrameParser::ReadOptimiser; - // end class list - - // type conventions: - - typedef uint32_t CellCount; // indications amount of cells. - typedef uint32_t CellID; // used for GIDs - typedef uint32_t CellIndex; // used for indexing arrays of cells.s - typedef uint32_t FrameCount; // indictions of amount of timesteps. - typedef uint32_t FrameIndex; // used for frame numbers - typedef int32_t - FrameDiff; // used for indicating time periods in terms of frame indexes (can be negative) - typedef double Time; // used for indicating biological time durations. - typedef uint64_t FrameDataCount; // used for indicating amount of data items for an entire - // frame - typedef uint64_t FrameDataIndex; // used for indexing arrays, vectors etc of data items. - typedef uint32_t CellDataCount; // used for indicating amount of data items that reside inside - // a single Cell. - typedef uint16_t SectionDataCount; // used for indicating amount of data items that reside - // inside a single section. - typedef uint16_t SectionIndex; // used for indexing data inside single sections. - typedef float MappingItem; // type in which SID and other mapping types are represented in. - typedef int32_t ExtraMappingItem; // type in which extra mapping items are represented in. - typedef float DataItem; // type in which the actual voltage, current, conductivity etc values - // are represented in - - enum SortingKey { SID, LOC }; // sorting specifier for sections. - // SID will make sections to be sorted by their mapping, - // LOC by their locaiton in file. - - namespace baseTypes { - // these typedefs define from which STL some of the clases are derived from - - typedef std::set FrameInfo; - typedef std::vector CompartmentMapping; - typedef std::vector SectionData; - // typedef std::map - // CellData; done blow - typedef std::map FrameOrganiser; - typedef std::vector ReadOptimiser; - } - - // end of type conventions - - class Header { - private: - bool isNative; - - // header info - uint32_t headerSize; - std::string libraryVersion; - std::string simulatorVersion; - CellCount amountOfCells; - FrameDataCount totalAmountOfCompartments; - FrameCount numberOfSteps; - Time startTime; - Time endTime; - Time dt; - std::string dataUnit; - std::string timeUnit; - FrameDataCount mappingSize; - std::string mappingName; - FrameDataCount extraMappingSize; - std::string extraMappingName; - std::string targetName; - - public: - Header() : headerSize(0){} - Header(char* readBuffer); - // Header(const Header& source); - - bool operator==(const Header& h1) const; - - // get methods: - inline bool sourceIsNative() const { - return isNative; - } - inline std::string getLibraryVersion() const { - return libraryVersion; - } - inline std::string getSimulatorVersion() const { - return simulatorVersion; - } - inline CellCount getNumberOfCells() const { - return amountOfCells; - } - inline FrameDataCount getTotalNumberOfCompartments() const { - return totalAmountOfCompartments; - } - inline FrameCount getNumberOfSteps() const { - return numberOfSteps; - } - inline std::string getMappingName() const { - return mappingName; - } - inline std::string getExtraMappingName() const { - return extraMappingName; - } - inline std::string getDataUnit() const { - return dataUnit; - } - inline std::string getTimeUnit() const { - return timeUnit; - } - inline FrameDataCount getMappingSize() const { - return mappingSize; - } - inline FrameDataCount getExtraMappingSize() const { - return extraMappingSize; - } - inline std::string getTargetName() const { - return targetName; - } - inline Time getStartTime() const { - return startTime; - } - inline Time getEndTime() const { - return endTime; - } - inline Time getTimeStepSize() const { - return dt; - } - // end get methods - }; - - class CellInfo { - private: - CellID cellNum; - CellDataCount amountOfCompartments; - - std::ios::off_type dataLocation; - std::ios::off_type extraMappingLocation; - std::ios::off_type mappingLocation; - - void* master; - - public: - enum key { GID, LOCATION }; +// class list: +class Header; +class CellInfo; +class FrameInfo; +class CompartmentMapping; +class SectionData; +// class SectionData::Identifier; +class Identifier; +class CellData; +class ReadManager; +class FrameParser; +// class FrameParser::FrameOrganiser; +// class FrameParser::ReadOptimiser; +// end class list + +// type conventions: + +typedef uint32_t CellCount; // indications amount of cells. +typedef uint32_t CellID; // used for GIDs +typedef uint32_t CellIndex; // used for indexing arrays of cells.s +typedef uint32_t FrameCount; // indictions of amount of timesteps. +typedef uint32_t FrameIndex; // used for frame numbers +typedef int32_t FrameDiff; // used for indicating time periods in terms of frame indexes (can be + // negative) +typedef double Time; // used for indicating biological time durations. +typedef uint64_t FrameDataCount; // used for indicating amount of data items for an entire + // frame +typedef uint64_t FrameDataIndex; // used for indexing arrays, vectors etc of data items. +typedef uint32_t CellDataCount; // used for indicating amount of data items that reside inside + // a single Cell. +typedef uint16_t SectionDataCount; // used for indicating amount of data items that reside + // inside a single section. +typedef uint16_t SectionIndex; // used for indexing data inside single sections. +typedef float MappingItem; // type in which SID and other mapping types are represented in. +typedef int32_t ExtraMappingItem; // type in which extra mapping items are represented in. +typedef float DataItem; // type in which the actual voltage, current, conductivity etc values + // are represented in + +enum SortingKey { SID, LOC }; // sorting specifier for sections. +// SID will make sections to be sorted by their mapping, +// LOC by their locaiton in file. + +namespace baseTypes { +// these typedefs define from which STL some of the clases are derived from + +typedef std::set FrameInfo; +typedef std::vector CompartmentMapping; +typedef std::vector SectionData; +// typedef std::map +// CellData; done blow +typedef std::map FrameOrganiser; +typedef std::vector ReadOptimiser; +} // namespace baseTypes + +// end of type conventions + +class Header +{ + private: + bool isNative; + + // header info + uint32_t headerSize; + std::string libraryVersion; + std::string simulatorVersion; + CellCount amountOfCells; + FrameDataCount totalAmountOfCompartments; + FrameCount numberOfSteps; + Time startTime; + Time endTime; + Time dt; + std::string dataUnit; + std::string timeUnit; + FrameDataCount mappingSize; + std::string mappingName; + FrameDataCount extraMappingSize; + std::string extraMappingName; + std::string targetName; + + public: + Header() + : headerSize(0) {} + Header(char* readBuffer); + // Header(const Header& source); + + bool operator==(const Header& h1) const; + + // get methods: + inline bool sourceIsNative() const { + return isNative; + } + inline std::string getLibraryVersion() const { + return libraryVersion; + } + inline std::string getSimulatorVersion() const { + return simulatorVersion; + } + inline CellCount getNumberOfCells() const { + return amountOfCells; + } + inline FrameDataCount getTotalNumberOfCompartments() const { + return totalAmountOfCompartments; + } + inline FrameCount getNumberOfSteps() const { + return numberOfSteps; + } + inline std::string getMappingName() const { + return mappingName; + } + inline std::string getExtraMappingName() const { + return extraMappingName; + } + inline std::string getDataUnit() const { + return dataUnit; + } + inline std::string getTimeUnit() const { + return timeUnit; + } + inline FrameDataCount getMappingSize() const { + return mappingSize; + } + inline FrameDataCount getExtraMappingSize() const { + return extraMappingSize; + } + inline std::string getTargetName() const { + return targetName; + } + inline Time getStartTime() const { + return startTime; + } + inline Time getEndTime() const { + return endTime; + } + inline Time getTimeStepSize() const { + return dt; + } + // end get methods +}; - CellInfo(); - CellInfo(char* readBuffer, bool isNative = true, key cmp = GID); +class CellInfo +{ + private: + CellID cellNum; + CellDataCount amountOfCompartments; - bool operator==(const CellInfo& c1) const; - bool operator!=(const CellInfo& c1) const; - bool operator<=(const CellInfo& c1) const; - bool operator>=(const CellInfo& c1) const; - bool operator<(const CellInfo& c1) const; - bool operator>(const CellInfo& c1) const; + std::ios::off_type dataLocation; + std::ios::off_type extraMappingLocation; + std::ios::off_type mappingLocation; - inline std::ios::off_type getDataLocation() const { - return dataLocation; - } + void* master; - inline std::ios::off_type getMappingLocation() const { - return mappingLocation; - } + public: + enum key { GID, LOCATION }; - inline std::ios::off_type getExtraMappingLocation() const { - return extraMappingLocation; - } + CellInfo(); + CellInfo(char* readBuffer, bool isNative = true, key cmp = GID); - inline CellID getCellNum() const { - return cellNum; - } + bool operator==(const CellInfo& c1) const; + bool operator!=(const CellInfo& c1) const; + bool operator<=(const CellInfo& c1) const; + bool operator>=(const CellInfo& c1) const; + bool operator<(const CellInfo& c1) const; + bool operator>(const CellInfo& c1) const; - inline CellDataCount getAmountOfCompartments() const { - return amountOfCompartments; - } + inline std::ios::off_type getDataLocation() const { + return dataLocation; + } - static bool compareByOffset(const CellInfo& c1, const CellInfo& c2); - }; + inline std::ios::off_type getMappingLocation() const { + return mappingLocation; + } - class FrameInfo : private baseTypes::FrameInfo { - private: - typedef baseTypes::FrameInfo Base; - typedef std::map Lookup; - Lookup gidLookup; + inline std::ios::off_type getExtraMappingLocation() const { + return extraMappingLocation; + } - public: - typedef Base::const_iterator const_iterator; + inline CellID getCellNum() const { + return cellNum; + } - FrameInfo() { - } - FrameInfo(char* buffer, - char* endOfBuffer, - bool isNative = true, - CellInfo::key sortMode = CellInfo::GID); - FrameInfo& filterGIDs(const std::vector& gids); - - /** - * Copy constructor. Needed to ensure set iterators reference the correct collection - */ - FrameInfo(const FrameInfo& rhs); - - /** - * Assignment operator. Needed to ensure set iterators reference the correct collection - */ - FrameInfo& operator=(const FrameInfo& rhs); - - inline const CellInfo& operator[](CellID gid) { - return *gidLookup[gid]; - } + inline CellDataCount getAmountOfCompartments() const { + return amountOfCompartments; + } - inline const_iterator begin() const { - return Base::begin(); - } + static bool compareByOffset(const CellInfo& c1, const CellInfo& c2); +}; + +class FrameInfo: private baseTypes::FrameInfo +{ + private: + typedef baseTypes::FrameInfo Base; + typedef std::map Lookup; + Lookup gidLookup; + + public: + typedef Base::const_iterator const_iterator; + + FrameInfo() {} + FrameInfo(char* buffer, + char* endOfBuffer, + bool isNative = true, + CellInfo::key sortMode = CellInfo::GID); + FrameInfo& filterGIDs(const std::vector& gids); + + /** + * Copy constructor. Needed to ensure set iterators reference the correct collection + */ + FrameInfo(const FrameInfo& rhs); + + /** + * Assignment operator. Needed to ensure set iterators reference the correct collection + */ + FrameInfo& operator=(const FrameInfo& rhs); + + inline const CellInfo& operator[](CellID gid) { + return *gidLookup[gid]; + } - inline const_iterator end() const { - return Base::end(); - } - }; + inline const_iterator begin() const { + return Base::begin(); + } - class CompartmentMapping : public baseTypes::CompartmentMapping { - private: - typedef baseTypes::CompartmentMapping Base; + inline const_iterator end() const { + return Base::end(); + } +}; - public: - typedef Base::size_type SizeType; +class CompartmentMapping: public baseTypes::CompartmentMapping +{ + private: + typedef baseTypes::CompartmentMapping Base; - CompartmentMapping(); - CompartmentMapping(MappingItem* start, MappingItem* end); - CompartmentMapping(MappingItem* buffer, SizeType bufSize); - }; + public: + typedef Base::size_type SizeType; - class SectionData : public baseTypes::SectionData { - private: - typedef baseTypes::SectionData Base; + CompartmentMapping(); + CompartmentMapping(MappingItem* start, MappingItem* end); + CompartmentMapping(MappingItem* buffer, SizeType bufSize); +}; - public: - class Identifier; - SectionData(); - SectionData(DataItem firstData); - SectionData(DataItem* start, DataItem* end); +class SectionData: public baseTypes::SectionData +{ + private: + typedef baseTypes::SectionData Base; - bool operator==(const SectionData& s1); - bool operator!=(const SectionData& s1); + public: + class Identifier; + SectionData(); + SectionData(DataItem firstData); + SectionData(DataItem* start, DataItem* end); - inline SectionDataCount numOfCompartments() { - return size(); - } + bool operator==(const SectionData& s1); + bool operator!=(const SectionData& s1); - static bool compareByOriginalLocation(const Identifier& c1, const Identifier& c2); - }; + inline SectionDataCount numOfCompartments() { + return size(); + } - class SectionData::Identifier { - public: - private: - typedef SectionData::Identifier thisType; + static bool compareByOriginalLocation(const Identifier& c1, const Identifier& c2); +}; - MappingItem sid; - SectionIndex loc; +class SectionData::Identifier +{ + public: + private: + typedef SectionData::Identifier thisType; - SortingKey key; + MappingItem sid; + SectionIndex loc; - public: - inline Identifier(const MappingItem& map, - const SectionIndex& location, - SortingKey primary = SID) { - sid = map, loc = location, key = primary; - } + SortingKey key; - inline bool operator!=(const thisType& c) const { - return (key == SID) ? sid != c.sid : loc != c.loc; - } + public: + inline Identifier(const MappingItem& map, + const SectionIndex& location, + SortingKey primary = SID) { + sid = map, loc = location, key = primary; + } - inline bool operator==(const thisType& c) const { - return (key == SID) ? sid == c.sid : loc == c.loc; - } + inline bool operator!=(const thisType& c) const { + return (key == SID) ? sid != c.sid : loc != c.loc; + } - inline bool operator<(const thisType& c) const { - return (key == SID) ? sid < c.sid : loc < c.loc; - } + inline bool operator==(const thisType& c) const { + return (key == SID) ? sid == c.sid : loc == c.loc; + } - inline bool operator<=(const thisType& c) const { - return (key == SID) ? sid <= c.sid : loc <= c.loc; - } + inline bool operator<(const thisType& c) const { + return (key == SID) ? sid < c.sid : loc < c.loc; + } - inline bool operator>(const thisType& c) const { - return (key == SID) ? sid > c.sid : loc > c.loc; - } + inline bool operator<=(const thisType& c) const { + return (key == SID) ? sid <= c.sid : loc <= c.loc; + } - inline bool operator>=(const thisType& c) const { - return (key == SID) ? sid >= c.sid : loc >= c.loc; - } + inline bool operator>(const thisType& c) const { + return (key == SID) ? sid > c.sid : loc > c.loc; + } - inline operator MappingItem() const { - return sid; - } + inline bool operator>=(const thisType& c) const { + return (key == SID) ? sid >= c.sid : loc >= c.loc; + } - inline MappingItem getSID() const { - return sid; - } + inline operator MappingItem() const { + return sid; + } - inline SectionIndex getLocation() const { - return loc; - } - }; + inline MappingItem getSID() const { + return sid; + } - namespace baseTypes { - typedef std::map CellData; + inline SectionIndex getLocation() const { + return loc; } +}; - class CellData : public baseTypes::CellData { - private: - typedef baseTypes::CellData Base; - typedef std::map Lookup; +namespace baseTypes { +typedef std::map CellData; +} - SortingKey sectSort; - Lookup sidLookup; +class CellData: public baseTypes::CellData +{ + private: + typedef baseTypes::CellData Base; + typedef std::map Lookup; - public: - CellData(); - CellData(const SortingKey segmentSorting = SID); - CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping); - void fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping); + SortingKey sectSort; + Lookup sidLookup; - inline iterator find(const SectionData::Identifier& i) { - return Base::find(i); - } + public: + CellData(); + CellData(const SortingKey segmentSorting = SID); + CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping); + void fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping); - inline iterator find(const MappingItem& map_find) { - Lookup::iterator match = sidLookup.find(map_find); - if (match == sidLookup.end()) - return end(); - else - return match->second; - } + inline iterator find(const SectionData::Identifier& i) { + return Base::find(i); + } - bool operator==(const CellData& d1); - }; + inline iterator find(const MappingItem& map_find) { + Lookup::iterator match = sidLookup.find(map_find); + if (match == sidLookup.end()) + return end(); + else + return match->second; + } - class ReadManager : public std::ifstream { - private: - typedef std::ifstream Base; + bool operator==(const CellData& d1); +}; - public: - typedef Base::off_type FileOffset; - typedef Base::pos_type FilePosition; +class ReadManager: public std::ifstream +{ + private: + typedef std::ifstream Base; - private: - FileOffset cellInfoOrigin; // position in file where cell info starts (after header) + public: + typedef Base::off_type FileOffset; + typedef Base::pos_type FilePosition; - bool isNative; // indicates whether file is originated from an architecture that arranges - // variables in same byte order + private: + FileOffset cellInfoOrigin; // position in file where cell info starts (after header) - FrameDataCount mappingSize_floats; + bool isNative; // indicates whether file is originated from an architecture that arranges + // variables in same byte order - FileOffset startLocation; - FileOffset mappingLocation; + FrameDataCount mappingSize_floats; - inline void seekgStart() { - seekg(startLocation, beg); - } + FileOffset startLocation; + FileOffset mappingLocation; - static CellInfo makeNullInfo(); + inline void seekgStart() { + seekg(startLocation, beg); + } - protected: - inline FileOffset getStartOffset() { - return startLocation; - } + static CellInfo makeNullInfo(); - public: - static const CellInfo _null_info; + protected: + inline FileOffset getStartOffset() { + return startLocation; + } - Header fileInfo; + public: + static const CellInfo _null_info; - ReadManager(); + Header fileInfo; - /** - * Constructor which immediately opens a file for reading. - * - * @param filePath The bbp binary report file to open - */ - explicit ReadManager(const std::string& filePath); + ReadManager(); - /** - * Opens the filename specified provided this object has not already opened a file - * - * @param filePath The bbp binary report file to open - */ - void open(const std::string& filePath); + /** + * Constructor which immediately opens a file for reading. + * + * @param filePath The bbp binary report file to open + */ + explicit ReadManager(const std::string& filePath); - // Find the cell's info which corresponds to cellNum (note: cellNum is not the cell index in - // the file) - CellInfo retrieveFindCell(CellID cellNum); + /** + * Opens the filename specified provided this object has not already opened a file + * + * @param filePath The bbp binary report file to open + */ + void open(const std::string& filePath); - // Retrieve global (time invariant) information of a cell (not actual data of the cell) - CellInfo retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode = CellInfo::LOCATION); + // Find the cell's info which corresponds to cellNum (note: cellNum is not the cell index in + // the file) + CellInfo retrieveFindCell(CellID cellNum); - // NOTE: its highly recommended to either disable cache (set to 0) or set cache size to - // unlimited (set to -1) before using this method: - FrameInfo retrieveAllCellInfo(CellInfo::key sortMode = CellInfo::LOCATION); + // Retrieve global (time invariant) information of a cell (not actual data of the cell) + CellInfo retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode = CellInfo::LOCATION); - /*seekg*: - * all the seekg functions below are wrapping around the original iostream seekg method. - */ + // NOTE: its highly recommended to either disable cache (set to 0) or set cache size to + // unlimited (set to -1) before using this method: + FrameInfo retrieveAllCellInfo(CellInfo::key sortMode = CellInfo::LOCATION); - inline void seekgCellInfo(CellIndex cellIndex = 0) { - seekg(cellInfoOrigin); - if (cellIndex) - seekg(cellIndex * SIZE_CELL_INFO_LENGTH, cur); - } + /*seekg*: + * all the seekg functions below are wrapping around the original iostream seekg method. + */ - inline void seekgData(const CellInfo& cellSpec, FrameIndex timeStep = 0) { - seekg(cellSpec.getDataLocation() + timeStep * getFrameSize_bytes(), beg); - } + inline void seekgCellInfo(CellIndex cellIndex = 0) { + seekg(cellInfoOrigin); + if (cellIndex) + seekg(cellIndex * SIZE_CELL_INFO_LENGTH, cur); + } - inline void seekgMapping(const CellInfo& cellSpec) { - seekg(cellSpec.getMappingLocation(), beg); - } + inline void seekgData(const CellInfo& cellSpec, FrameIndex timeStep = 0) { + seekg(cellSpec.getDataLocation() + timeStep * getFrameSize_bytes(), beg); + } - inline void seekgExtraMapping(const CellInfo& cellSpec) { - seekg(cellSpec.getExtraMappingLocation(), beg); - } + inline void seekgMapping(const CellInfo& cellSpec) { + seekg(cellSpec.getMappingLocation(), beg); + } - inline void seekgMappingStart() { - seekg(mappingLocation, beg); - } + inline void seekgExtraMapping(const CellInfo& cellSpec) { + seekg(cellSpec.getExtraMappingLocation(), beg); + } - inline void seekgFrame(FrameIndex timeStep) { - seekg(startLocation + getFrameSize_bytes() * timeStep); - } + inline void seekgMappingStart() { + seekg(mappingLocation, beg); + } - /*read(): - * this function essentially takes 3 arguments (the third being the datatype). - * it does the same as the read function of the base ifstream class that it is overloading - * the only difference is that num is not neccesarily in number of bytes - * instead it defines the mount of elemetns in the buffer (depending on the datatype of the - * buffer). - * - * Important: - * This read method will automatically fix the byte orders of the data if applicable. - * The algorithm of fixing the byte orders depends on the size of each element in the buffer - * array. - * Therefore you should be carefull with passing char* buffers, - * unless you are really intending to read single byte data. - */ - - template - ReadManager& read(datatype* buffer, std::streamsize num) { - Base::read((char*)buffer, num * sizeof(datatype)); - - if (sizeof(datatype) != 1 && !isNative) // a good compiler should not include this - // entire if statement when this function is - // reading single byte items. - // This if is to avoid entering an empty loop. Otherwise could have been done in - // switch below. - for (datatype* endOfBuffer = buffer + num; buffer < endOfBuffer; buffer++) - switch (sizeof(datatype)) { // metamorphic switch - case 8: // double, int64_t, offsets etc - DOUBLESWAP(*buffer); - break; - case 4: // float, int32_t, etc - FIX_INT(*buffer); - break; - case 2: // short - FIX_SHORT(*buffer); - break; - default: // anything weird that cannot be fixed. - // TODO: throw error exception - break; - } - return *this; - } + inline void seekgFrame(FrameIndex timeStep) { + seekg(startLocation + getFrameSize_bytes() * timeStep); + } - /*checkState(): - * will determine whether a reading error has occured, - * output the according error message if there has been an error, - * and exit execution if an error occured. - */ + /*read(): + * this function essentially takes 3 arguments (the third being the datatype). + * it does the same as the read function of the base ifstream class that it is overloading + * the only difference is that num is not neccesarily in number of bytes + * instead it defines the mount of elemetns in the buffer (depending on the datatype of the + * buffer). + * + * Important: + * This read method will automatically fix the byte orders of the data if applicable. + * The algorithm of fixing the byte orders depends on the size of each element in the buffer + * array. + * Therefore you should be carefull with passing char* buffers, + * unless you are really intending to read single byte data. + */ + + template + ReadManager& read(datatype* buffer, std::streamsize num) { + Base::read((char*) buffer, num * sizeof(datatype)); + + if (sizeof(datatype) != 1 && !isNative) // a good compiler should not include this + // entire if statement when this function is + // reading single byte items. + // This if is to avoid entering an empty loop. Otherwise could have been done in + // switch below. + for (datatype* endOfBuffer = buffer + num; buffer < endOfBuffer; buffer++) + switch (sizeof(datatype)) { // metamorphic switch + case 8: // double, int64_t, offsets etc + DOUBLESWAP(*buffer); + break; + case 4: // float, int32_t, etc + FIX_INT(*buffer); + break; + case 2: // short + FIX_SHORT(*buffer); + break; + default: // anything weird that cannot be fixed. + // TODO: throw error exception + break; + } + return *this; + } - void checkState(); + /*checkState(): + * will determine whether a reading error has occured, + * output the according error message if there has been an error, + * and exit execution if an error occured. + */ - /*getFrameSize: - * returns the number of bytes that hold the data of all compartments of all cells for a - * single timestep. - * please note that its is in bytes, not floats! - * use fileInfo.getTotalNumberOfCompartments() instead for number of floats. - */ + void checkState(); - inline std::streamsize getFrameSize_bytes() const { - return fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem); - } + /*getFrameSize: + * returns the number of bytes that hold the data of all compartments of all cells for a + * single timestep. + * please note that its is in bytes, not floats! + * use fileInfo.getTotalNumberOfCompartments() instead for number of floats. + */ - /*getMappingSize: - * returns the number of bytes that hold the data of the mappings of all compartmetns of all - * cells. - * please note that this is in bytes not floats. - */ + inline std::streamsize getFrameSize_bytes() const { + return fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem); + } - inline std::streamsize getMappingSize_bytes() const { - return mappingSize_floats * sizeof(MappingItem); - } + /*getMappingSize: + * returns the number of bytes that hold the data of the mappings of all compartmetns of all + * cells. + * please note that this is in bytes not floats. + */ - // The following are low level c-style reading interfaces for max performance - they provide - // unsorted, raw data. - /*readFrameMapping: - * will fill buffer with mappings of all compartmetns of all cells. - * the file pointer will be automatically adjusted accordingly and then restored to original - * position. - * use getMappingSize() / sizeof(float) to determine the size of the buffer in number bytes - * (NOT float). - */ - - void readFrameMapping(MappingItem* buffer); - - /*readFrame: - * will fill dataBuffer with data from next frame. - * use seekgFrame() to specify timestep. - * use fileInfo.getTotalNumberOfCompartments() to determine the size of buffer - */ - void readFrame(DataItem* dataBuffer); + inline std::streamsize getMappingSize_bytes() const { + return mappingSize_floats * sizeof(MappingItem); + } + // The following are low level c-style reading interfaces for max performance - they provide + // unsorted, raw data. + /*readFrameMapping: + * will fill buffer with mappings of all compartmetns of all cells. + * the file pointer will be automatically adjusted accordingly and then restored to original + * position. + * use getMappingSize() / sizeof(float) to determine the size of the buffer in number bytes + * (NOT float). + */ + + void readFrameMapping(MappingItem* buffer); + + /*readFrame: + * will fill dataBuffer with data from next frame. + * use seekgFrame() to specify timestep. + * use fileInfo.getTotalNumberOfCompartments() to determine the size of buffer + */ + void readFrame(DataItem* dataBuffer); + + private: + /** + * Upon opening a report file, read the header with metadata + */ + void readHeader(); +}; + +class FrameParser: protected ReadManager +{ + public: + typedef std::vector> CompartmentIndexing; + typedef std::vector> CompartmentCounts; + + protected: + class FrameOrganiser: public baseTypes::FrameOrganiser + { private: - /** - * Upon opening a report file, read the header with metadata - */ - void readHeader(); - }; + SortingKey sectSort; - class FrameParser : protected ReadManager { public: - typedef std::vector > CompartmentIndexing; - typedef std::vector > CompartmentCounts; + FrameOrganiser(); + explicit FrameOrganiser(SortingKey sectionSorting); - protected: - class FrameOrganiser : public baseTypes::FrameOrganiser { - private: - SortingKey sectSort; + void fillData(const FrameInfo& info, + const std::vector& mapping, + const float* dataBuffer); - public: - FrameOrganiser(); - explicit FrameOrganiser(SortingKey sectionSorting); + void makeOrderedDataBuffer(DataItem* buffer) const; - void fillData(const FrameInfo& info, - const std::vector& mapping, - const float* dataBuffer); + CompartmentIndexing makeOrderedOffsetReferences() const; + CompartmentCounts makeOrderedSegSizes() const; + }; - void makeOrderedDataBuffer(DataItem* buffer) const; + class ReadOptimiser: public baseTypes::ReadOptimiser + { + private: + void updateOffsets(size_t newSize); - CompartmentIndexing makeOrderedOffsetReferences() const; - CompartmentCounts makeOrderedSegSizes() const; - }; + protected: + FileOffset firstOffset; + FrameDataCount dataSize_elements; - class ReadOptimiser : public baseTypes::ReadOptimiser { - private: - void updateOffsets(size_t newSize); + size_t elementSize; + static size_t defaultElementSize; - protected: - FileOffset firstOffset; - FrameDataCount dataSize_elements; + public: + ReadOptimiser(); + explicit ReadOptimiser(std::streamsize frameSize_bytes); + ReadOptimiser(const FrameInfo& cellsToRead, + std::streamsize frameSize_bytes, + FilePosition startLocation); - size_t elementSize; - static size_t defaultElementSize; + inline FileOffset getFirst() const { + return firstOffset; + } - public: - ReadOptimiser(); - explicit ReadOptimiser(std::streamsize frameSize_bytes); - ReadOptimiser(const FrameInfo& cellsToRead, - std::streamsize frameSize_bytes, - FilePosition startLocation); + inline FrameDataCount getDataSize_elements() const { + return dataSize_elements; + } - inline FileOffset getFirst() const { - return firstOffset; + inline void setElementSize(size_t newSize) { + if (newSize != elementSize) { + updateOffsets(newSize); + elementSize = newSize; } + } - inline FrameDataCount getDataSize_elements() const { - return dataSize_elements; - } + static inline void setDefaultElementSize(size_t newDefault) { + defaultElementSize = newDefault; + } - inline void setElementSize(size_t newSize) { - if (newSize != elementSize) { - updateOffsets(newSize); - elementSize = newSize; - } - } + inline size_t getElementSize() const { + return elementSize; + } + }; - static inline void setDefaultElementSize(size_t newDefault) { - defaultElementSize = newDefault; - } + void readFrame(DataItem* buffer); - inline size_t getElementSize() const { - return elementSize; - } - }; + private: + DataItem* readBuffer; + DataItem** refArray; - void readFrame(DataItem* buffer); + ReadOptimiser offsets; + FrameOrganiser org; - private: - DataItem* readBuffer; - DataItem** refArray; + FrameIndex timestep; - ReadOptimiser offsets; - FrameOrganiser org; + void createReferences(const FrameInfo& cinfo, bool sortData); - FrameIndex timestep; + void bufferTransfer(DataItem* target) const; - void createReferences(const FrameInfo& cinfo, bool sortData); + public: + /*Constructor: + * Will read next frame of data and parse it into a map of CellData structure from file. + */ + FrameParser(); + FrameParser(const std::string& file, bool sortData = true); + FrameParser(const std::string& file, + const std::vector& gidTarget, + bool sortData = true); + ~FrameParser(); - void bufferTransfer(DataItem* target) const; + /*retarget: + * Takes a list of gids in form of a vector of CellID and uses it to respecify the target + * cells of this + * frameparser. + */ - public: - /*Constructor: - * Will read next frame of data and parse it into a map of CellData structure from file. - */ - FrameParser(); - FrameParser(const std::string& file, bool sortData = true); - FrameParser(const std::string& file, - const std::vector& gidTarget, - bool sortData = true); - ~FrameParser(); - - /*retarget: - * Takes a list of gids in form of a vector of CellID and uses it to respecify the target - * cells of this - * frameparser. - */ - - inline FrameIndex simtime2index(Time time) { - return (FrameIndex)((time - fileInfo.getStartTime()) / fileInfo.getTimeStepSize()); - } + inline FrameIndex simtime2index(Time time) { + return (FrameIndex)((time - fileInfo.getStartTime()) / fileInfo.getTimeStepSize()); + } - void retarget(const std::vector& gidTarget, bool sortData = true); + void retarget(const std::vector& gidTarget, bool sortData = true); - inline const Header& getHeader() const { - return fileInfo; - } + inline const Header& getHeader() const { + return fileInfo; + } - inline FrameDataCount getBufferSize_elements() const { - return offsets.getDataSize_elements(); - } + inline FrameDataCount getBufferSize_elements() const { + return offsets.getDataSize_elements(); + } - // to select frame (similar use to iterators): - // Note that readFrameData already incorporates operator++ ! - FrameParser& operator++(); // increment timestep - FrameParser& operator++(int); // increment timestep - FrameParser& operator--(); // decrement timestep - FrameParser& operator--(int); // decrement timestep - FrameParser& operator+=(FrameDiff increment); // increment timestep by scalar - FrameParser& operator-=(FrameDiff decrement); // decrement timestep by scalar - FrameParser& operator=(FrameIndex newTime); // set timestep to scalar - - // returns the current timestep: - inline FrameIndex getTimestep() const { - return timestep; - } + // to select frame (similar use to iterators): + // Note that readFrameData already incorporates operator++ ! + FrameParser& operator++(); // increment timestep + FrameParser& operator++(int); // increment timestep + FrameParser& operator--(); // decrement timestep + FrameParser& operator--(int); // decrement timestep + FrameParser& operator+=(FrameDiff increment); // increment timestep by scalar + FrameParser& operator-=(FrameDiff decrement); // decrement timestep by scalar + FrameParser& operator=(FrameIndex newTime); // set timestep to scalar + + // returns the current timestep: + inline FrameIndex getTimestep() const { + return timestep; + } - inline bool hasMore() const { - return timestep < fileInfo.getNumberOfSteps(); - } + inline bool hasMore() const { + return timestep < fileInfo.getNumberOfSteps(); + } - // returns the current time that the current timestep represents. - Time getTime() const; - - void readFrameMapping(MappingItem* buffer); - void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame - // after. Data is sorted by GID and segment num - - /*getReferences() - * returns a vector of vectors of offsets that correspond to the relevant segments inside - * the read buffer. - * eg. - * getReferences()[12][31]; //will give the offset for segment 31 in cell 12 - * if the segment has no comparments, it will return an offset of 0. - */ - inline CompartmentIndexing getReferences() const { - return org.makeOrderedOffsetReferences(); - } + // returns the current time that the current timestep represents. + Time getTime() const; + + void readFrameMapping(MappingItem* buffer); + void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame + // after. Data is sorted by GID and segment num + + /*getReferences() + * returns a vector of vectors of offsets that correspond to the relevant segments inside + * the read buffer. + * eg. + * getReferences()[12][31]; //will give the offset for segment 31 in cell 12 + * if the segment has no comparments, it will return an offset of 0. + */ + inline CompartmentIndexing getReferences() const { + return org.makeOrderedOffsetReferences(); + } - /*getCompartmentCounts() - * returns a vector of vectors like getReferences() above, - * but instead of containing the offsets for each segment, - * it contains the amount of compartmetns of the corresponding segment. - */ - inline CompartmentCounts getCompartmentCounts() const { - return org.makeOrderedSegSizes(); - } - }; -} + /*getCompartmentCounts() + * returns a vector of vectors like getReferences() above, + * but instead of containing the offsets for each segment, + * it contains the amount of compartmetns of the corresponding segment. + */ + inline CompartmentCounts getCompartmentCounts() const { + return org.makeOrderedSegSizes(); + } +}; +} // namespace bbpReader #endif diff --git a/tools/converter/binary_reader/crossPlatformConversion.h b/tools/converter/binary_reader/crossPlatformConversion.h index 7e33343..b14b9ee 100644 --- a/tools/converter/binary_reader/crossPlatformConversion.h +++ b/tools/converter/binary_reader/crossPlatformConversion.h @@ -6,14 +6,14 @@ #define SMALL_NUM 0.00000001 // anything that avoids division overflow #define ABS(x) ((x) >= 0 ? (x) : -(x)) // absolute value -#define SWAP_2(x) ((((x)&0xff) << 8) | ((unsigned short)(x) >> 8)) +#define SWAP_2(x) ((((x) &0xff) << 8) | ((unsigned short) (x) >> 8)) #define SWAP_4(x) \ (((x) << 24) | (((x) << 8) & 0x00ff0000) | (((x) >> 8) & 0x0000ff00) | ((x) >> 24)) -#define FIX_SHORT(x) (*(unsigned short*)&(x) = SWAP_2(*(unsigned short*)&(x))) -#define FIX_INT(x) (*(unsigned int*)&(x) = SWAP_4(*(unsigned int*)&(x))) +#define FIX_SHORT(x) (*(unsigned short*) &(x) = SWAP_2(*(unsigned short*) &(x))) +#define FIX_INT(x) (*(unsigned int*) &(x) = SWAP_4(*(unsigned int*) &(x))) #define FIX_LONG(x) FIX_INT(x) #define FIX_FLOAT(x) FIX_INT(x) -#define DOUBLESWAP(x) ByteSwap((unsigned char*)&x, sizeof(x)) +#define DOUBLESWAP(x) ByteSwap((unsigned char*) &x, sizeof(x)) inline void ByteSwap(unsigned char* b, int n) { int i = 0; diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index 52a2050..9c6a317 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -90,7 +90,6 @@ int main(int argc, char* argv[]) { // Generate the initial structure sonata_prepare_datasets(); - uint32_t num_steps = header.getNumberOfSteps(); uint64_t element_ids_per_frame = frame_parser.getBufferSize_elements(); DataItem* element_ids_buffer = new DataItem[element_ids_per_frame]; uint32_t timestep = 0; From 35e00de793ad78a515c7d00258d15f5363814794 Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Mon, 10 Oct 2022 18:03:23 +0200 Subject: [PATCH 08/10] Add optional parameter 'population_name' --- tools/converter/reports_converter.cpp | 29 ++++++++++++++++++--------- tools/converter/spikes_converter.cpp | 15 +++++++++----- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index 9c6a317..ff95854 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -15,13 +15,15 @@ using namespace bbpReader; static void show_usage(std::string name) { - std::cerr << "Usage: " << name << " \n" + std::cerr << "Usage: " << name << " [population_name]\n" << "Options:\n" - << "\t-h,--help\t\t\tShow this help message\n" - << "\t<[--soma, --compartment]>\tSelect soma or compartment report\n" + << "\t-h,--help\t\tShow this help message\n" + << "\treport_filename\t\tName of the report to convert\n" + << "\treport_type\t\t'--soma' or '--compartment' required\n" + << "\tpopulation_name\t\tName of the report population (Default 'All')\n" << "Examples:\n" - << "\t " << name << " soma.bbp --soma\n" - << "\t " << name << " compartment.bbp --compartment" << std::endl; + << "\t " << name << " soma.bbp --soma PopulationA\n" + << "\t " << name << " compartment.bbp --compartment PopulationB" << std::endl; } bool help(int argc, char* argv[]) { @@ -38,12 +40,20 @@ int main(int argc, char* argv[]) { #ifdef SONATA_REPORT_HAVE_MPI MPI_Init(nullptr, nullptr); #endif - if (argc != 3 || help(argc, argv)) { + if ((argc != 3 && argc != 4) || help(argc, argv)) { show_usage(argv[0]); return -1; } const std::string file_name = argv[1]; const std::string report_type = argv[2]; + if (report_type != "--soma" && report_type != "--compartment") { + logger->error("report type: --soma or --compartment required"); + return -3; + } + std::string population_name = "All"; + if (argc == 4) { + population_name = argv[3]; + } logger->info("Trying to convert '{}' binary report...'", file_name); std::ifstream f(file_name); if (!f.good()) { @@ -70,10 +80,10 @@ int main(int argc, char* argv[]) { std::vector node_ids(1); for (FrameInfo::const_iterator it = cells.begin(); it != cells.end(); ++it) { node_ids[0] = it->getCellNum(); - sonata_add_node(report_name.data(), "All", 0, it->getCellNum()); + sonata_add_node(report_name.data(), population_name.data(), 0, it->getCellNum()); if (report_type == "--soma") { - sonata_add_element(report_name.data(), "All", node_ids[0], 0, nullptr); + sonata_add_element(report_name.data(), population_name.data(), node_ids[0], 0, nullptr); } else { // --compartment FrameParser frame_parser_gid(file_name, node_ids); int num_element_ids = frame_parser_gid.getBufferSize_elements(); @@ -83,7 +93,8 @@ int main(int argc, char* argv[]) { std::vector element_ids(element_ids_buffer, element_ids_buffer + num_element_ids); for (auto& element : element_ids) { - sonata_add_element(report_name.data(), "All", node_ids[0], element, nullptr); + sonata_add_element( + report_name.data(), population_name.data(), node_ids[0], element, nullptr); } } } diff --git a/tools/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp index 9c4ae34..4a1e147 100644 --- a/tools/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -11,11 +11,13 @@ #include static void show_usage(std::string name) { - std::cerr << "Usage: " << name << " \n" + std::cerr << "Usage: " << name << " [population_name]\n" << "Options:\n" - << "\t-h,--help\tShow this help message\n" + << "\t-h,--help\t\tShow this help message\n" + << "\tspike_filename\t\tName of the report to convert\n" + << "\tpopulation_name\t\tName of the report population (Default 'All')\n" << "Example:\n" - << "\t " << name << " out.dat" << std::endl; + << "\t " << name << " out.dat PopulationA" << std::endl; } bool help(int argc, char* argv[]) { @@ -32,12 +34,16 @@ int main(int argc, char* argv[]) { #ifdef SONATA_REPORT_HAVE_MPI MPI_Init(nullptr, nullptr); #endif - if (argc != 2 || help(argc, argv)) { + if ((argc != 2 && argc != 3) || help(argc, argv)) { show_usage(argv[0]); return -1; } const std::string file_name = argv[1]; + std::string population_name = "All"; + if (argc == 3) { + population_name = argv[2]; + } logger->info("Trying to convert '{}' binary report...'", file_name); std::ifstream infile(file_name); if (!infile) { @@ -66,7 +72,6 @@ int main(int argc, char* argv[]) { std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); sonata_create_spikefile(".", report_name.data()); - std::string population_name = "All"; uint64_t population_offset = 0; sonata_add_spikes_population(population_name.data(), population_offset, From 4846d589f672f4ba52100cb26e739c62447572ee Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Tue, 11 Oct 2022 15:22:34 +0200 Subject: [PATCH 09/10] General reader refactoring - CamelCase to snake_case - file name changes --- .../converter/binary_reader/BinaryPositions.h | 62 -- tools/converter/binary_reader/CMakeLists.txt | 2 +- tools/converter/binary_reader/ReadManager.cpp | 877 ----------------- .../binary_reader/binary_positions.h | 62 ++ .../converter/binary_reader/binary_reader.cpp | 883 ++++++++++++++++++ .../{ReadManager.h => binary_reader.h} | 404 ++++---- ...nversion.h => cross_platform_conversion.h} | 5 +- tools/converter/reports_converter.cpp | 28 +- 8 files changed, 1161 insertions(+), 1162 deletions(-) delete mode 100644 tools/converter/binary_reader/BinaryPositions.h delete mode 100644 tools/converter/binary_reader/ReadManager.cpp create mode 100644 tools/converter/binary_reader/binary_positions.h create mode 100644 tools/converter/binary_reader/binary_reader.cpp rename tools/converter/binary_reader/{ReadManager.h => binary_reader.h} (57%) rename tools/converter/binary_reader/{crossPlatformConversion.h => cross_platform_conversion.h} (87%) diff --git a/tools/converter/binary_reader/BinaryPositions.h b/tools/converter/binary_reader/BinaryPositions.h deleted file mode 100644 index 28a7a9d..0000000 --- a/tools/converter/binary_reader/BinaryPositions.h +++ /dev/null @@ -1,62 +0,0 @@ -// FixedPositions // Don't change so we can have backward compatibility - -// If identifier read at position 0 matches ARCHITECTURE_IDENTIFIER, then the file was writting from -// native architecture -#define ARCHITECTURE_IDENTIFIER 1.001 - -// Double: -#define IDENTIFIER_POSITION 0 -// Int: -#define HEADER_SIZE_POSITION IDENTIFIER_POSITION + sizeof(double) -// String: -#define LIBRARY_VERSION_POSITION 16 -// String: -#define SIMULATOR_VERSION_POSITION 32 -// Int: -#define TOTAL_NUMBER_OF_CELLS_POSITION 48 -// Int: -#define TOTAL_NUMBER_OF_COMPARTMENTS_POSITION 52 -// Int: -#define NUMBER_OF_STEPS_POSITION 64 -// Double: -#define TIME_START_POSITION 72 -// Double: -#define TIME_END_POSITION 80 -// Double: -#define DT_TIME_POSITION 88 -// String: -#define D_UNIT_POSITION 96 -// String: -#define T_UNIT_POSITION 112 -// Int: -#define MAPPING_SIZE_POSITION 128 -// String: -#define MAPPING_NAME_POSITION 144 -// Int: -#define EXTRA_MAPPING_SIZE_POSITION 160 -// String: -#define EXTRA_MAPPING_NAME_POSITION 176 -// String: -#define TARGET_NAME_POSITION 192 - -// Length definition -// int: -#define HEADER_LENGTH 1024 -// int: -#define SIZE_CELL_INFO_LENGTH 64 -// int: -#define SIZE_STRING 16 - -// Inside CellInfo -// Int: -#define NUMBER_OF_CELL_POSITION 0 -// Int: -#define NUMBER_OF_COMPARTMENTS_POSITION 8 -// OffSet: -#define DATA_INFO_POSITION 16 -// OffSet: -#define EXTRA_MAPPING_INFO_POSITION 24 -// OffSet: -#define MAPPING_INFO_POSITION 32 - -// order of the cell is a simple string order diff --git a/tools/converter/binary_reader/CMakeLists.txt b/tools/converter/binary_reader/CMakeLists.txt index 61ab63f..c92acf3 100644 --- a/tools/converter/binary_reader/CMakeLists.txt +++ b/tools/converter/binary_reader/CMakeLists.txt @@ -1,3 +1,3 @@ -set(READER_SOURCES ReadManager.cpp) +set(READER_SOURCES binary_reader.cpp) add_library (binary_reader STATIC ${READER_SOURCES}) diff --git a/tools/converter/binary_reader/ReadManager.cpp b/tools/converter/binary_reader/ReadManager.cpp deleted file mode 100644 index b29aee3..0000000 --- a/tools/converter/binary_reader/ReadManager.cpp +++ /dev/null @@ -1,877 +0,0 @@ -#include "ReadManager.h" -#include -#include - -using namespace std; -namespace bbpReader { - -Header::Header(char* readBuffer) { -#define fetch(TYPE, POSITION) *((TYPE*) (readBuffer + POSITION)) -#define fetchS(POSITION) string((char*) (readBuffer + POSITION)) - - double readIdentifier = fetch(double, IDENTIFIER_POSITION); - headerSize = fetch(int32_t, HEADER_SIZE_POSITION); - amountOfCells = fetch(int32_t, TOTAL_NUMBER_OF_CELLS_POSITION); - /** \bug 32 bit is consistent with the binary format specification of the - header, however - */ - uint32_t compartmentsPerFrame = fetch(int32_t, TOTAL_NUMBER_OF_COMPARTMENTS_POSITION); - libraryVersion = fetchS(LIBRARY_VERSION_POSITION); - simulatorVersion = fetchS(SIMULATOR_VERSION_POSITION); - numberOfSteps = fetch(int32_t, NUMBER_OF_STEPS_POSITION); - startTime = fetch(double, TIME_START_POSITION); - endTime = fetch(double, TIME_END_POSITION); - dt = fetch(double, DT_TIME_POSITION); - dataUnit = fetchS(D_UNIT_POSITION); - timeUnit = fetchS(T_UNIT_POSITION); - mappingSize = fetch(int32_t, MAPPING_SIZE_POSITION); - mappingName = fetchS(MAPPING_NAME_POSITION); - extraMappingSize = fetch(int32_t, EXTRA_MAPPING_SIZE_POSITION); - extraMappingName = fetchS(EXTRA_MAPPING_NAME_POSITION); - targetName = fetchS(TARGET_NAME_POSITION); - -#undef fetchS -#undef fetch - - isNative = readIdentifier == ARCHITECTURE_IDENTIFIER; - - if (!isNative) { - // check if data can be converted: - DOUBLESWAP(readIdentifier); - if (readIdentifier != ARCHITECTURE_IDENTIFIER) { - cerr << "File is corrupt or originated from an unknown architecture." << endl; - exit(1); - } - - // if data was stored on a machine with different architecture, - // then the data needs to be converted to be compatible with the reading machine - FIX_INT(headerSize); - FIX_INT(amountOfCells); - FIX_INT(compartmentsPerFrame); - FIX_INT(numberOfSteps); - DOUBLESWAP(startTime); - DOUBLESWAP(endTime); - DOUBLESWAP(dt); - FIX_INT(mappingSize); - FIX_INT(extraMappingSize); - } - - totalAmountOfCompartments = compartmentsPerFrame; -} - -bool Header::operator==(const Header& h1) const { -#define _CHECK(field) \ - if (h1.field != field) \ - return false - // compare all fields other than isNative - _CHECK(headerSize); - _CHECK(libraryVersion); - _CHECK(simulatorVersion); - _CHECK(amountOfCells); - _CHECK(totalAmountOfCompartments); - _CHECK(numberOfSteps); - _CHECK(startTime); - _CHECK(endTime); - _CHECK(dt); - _CHECK(dataUnit); - _CHECK(timeUnit); - _CHECK(mappingSize); - _CHECK(mappingName); - _CHECK(extraMappingName); - _CHECK(extraMappingSize); - _CHECK(targetName); - - // else if nothing returned false: - return true; -#undef _CHECK -} - -CellInfo::CellInfo() { - cellNum = 0; - master = &cellNum; - amountOfCompartments = 0; - dataLocation = 0; - extraMappingLocation = 0; - mappingLocation = 0; -} - -CellInfo::CellInfo(char* readBuffer, bool isNative, key cmp) { -#define fetch(TYPE, POSITION) *((TYPE*) (readBuffer + POSITION)) - cellNum = fetch(int32_t, NUMBER_OF_CELL_POSITION); - amountOfCompartments = fetch(int32_t, NUMBER_OF_COMPARTMENTS_POSITION); - - dataLocation = fetch(uint64_t, DATA_INFO_POSITION); - extraMappingLocation = fetch(uint64_t, EXTRA_MAPPING_INFO_POSITION); - mappingLocation = fetch(uint64_t, MAPPING_INFO_POSITION); - -#undef fetch - - if (!isNative) { - FIX_INT(cellNum); - FIX_INT(amountOfCompartments); - - DOUBLESWAP(dataLocation); - DOUBLESWAP(extraMappingLocation); - DOUBLESWAP(mappingLocation); - } - - if (cmp == GID) - master = &cellNum; - else - master = &dataLocation; -} - -bool CellInfo::operator==(const CellInfo& c1) const { -#define _CHECK(field) \ - if (c1.field != field) \ - return false - _CHECK(cellNum); - _CHECK(amountOfCompartments); - _CHECK(dataLocation); - _CHECK(extraMappingLocation); - _CHECK(mappingLocation); - - return true; -#undef _CHECK -} - -bool CellInfo::operator!=(const CellInfo& c1) const { - return !(*this == c1); -} - -#define COMPARE_(op) (master == &cellNum) ? cellNum op c.cellNum : dataLocation op c.dataLocation - -#define DEFINE_CMP_OP(op) \ - bool CellInfo::operator op(const CellInfo& c) const { \ - return COMPARE_(op); \ - } - -DEFINE_CMP_OP(<) -DEFINE_CMP_OP(<=) -DEFINE_CMP_OP(>) -DEFINE_CMP_OP(>=) - -#undef DEFINE_CMP_OP -#undef COMPARE_ - -bool CellInfo::compareByOffset(const CellInfo& a, const CellInfo& b) { - return a.dataLocation < b.dataLocation; -} - -FrameInfo::FrameInfo(char* buffer, char* endOfBuffer, bool isNative, CellInfo::key sortMode) { - // create a cellinfo and then insert into itself (rem, this class inherits from set). The - // resulting set iterator is then put into - // a map member for associative lookup gid:location - for (char* bufferLoc = buffer; bufferLoc < endOfBuffer; bufferLoc += SIZE_CELL_INFO_LENGTH) { - iterator ins = insert(CellInfo(bufferLoc, isNative, sortMode)).first; - gidLookup.insert(make_pair(ins->getCellNum(), ins)); - } -} - -FrameInfo::FrameInfo(const FrameInfo& rhs) - : baseTypes::FrameInfo() { - *this = rhs; -} - -FrameInfo& FrameInfo::operator=(const FrameInfo& rhs) { - insert(rhs.begin(), rhs.end()); - for (iterator cellItem = begin(); cellItem != end(); ++cellItem) { - gidLookup.insert(make_pair(cellItem->getCellNum(), cellItem)); - } - - return *this; -} - -FrameInfo& FrameInfo::filterGIDs(const vector& gids) { - Lookup newGidLookup; - - for (vector::const_iterator gid = gids.begin(); gid != gids.end(); gid++) { - Lookup::iterator it = gidLookup.find(*gid); - if (it != gidLookup.end()) { - newGidLookup.insert(*it); - gidLookup.erase(it); - } else - throw string("GID doesn't exist in file."); // TODO: replace by custom std::exception - } - - // gidLookup contains the gids that need to be removed and the newGidLookup contains the - // gids that remain. - - for (Lookup::iterator i = gidLookup.begin(); i != gidLookup.end(); i++) - erase(i->second); // delete the unwanted gids. - - gidLookup.swap(newGidLookup); - - return *this; -} - -CompartmentMapping::CompartmentMapping() - : Base() {} - -CompartmentMapping::CompartmentMapping(MappingItem* start, MappingItem* _end) - : Base(start, _end) {} - -CompartmentMapping::CompartmentMapping(MappingItem* buffer, SizeType bufSize) - : Base(buffer, buffer + bufSize) {} - -SectionData::SectionData() - : Base() {} - -SectionData::SectionData(DataItem firstData) { - push_back(firstData); -} - -SectionData::SectionData(DataItem* start, DataItem* _end) - : Base(start, _end) {} - -bool SectionData::operator==(const SectionData& s1) { - if (s1.size() != size()) - return false; - - const_iterator dat1, dat2; - for (dat1 = s1.begin(), dat2 = begin(); dat2 != end(); dat1++, dat2++) - if (*dat1 != *dat2) - return false; - - return true; -} - -bool SectionData::operator!=(const SectionData& s1) { - return !(*this == s1); -} - -bool SectionData::compareByOriginalLocation(const Identifier& c1, const Identifier& c2) { - return c1.getLocation() < c2.getLocation(); -} - -CellData::CellData() { - sectSort = SID; -} - -CellData::CellData(SortingKey sorting) { - sectSort = sorting; -} - -CellData::CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { - sectSort = SID; - fillData(dataBuffer, mapping); -} - -void CellData::fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping) { - CompartmentMapping::const_iterator it; - SectionIndex i; - for (it = mapping.begin(), i = 0; it != mapping.end(); it++, dataBuffer++, i++) { - // iteration through all compartments inside cell - DataItem data = *dataBuffer; - - iterator segMatch = find(*it); // find corresponding segment - - if (segMatch == end()) { - // SID was not found. - // this means it wasnt created yet. - // therefore create a new one and insert first data item into it: - iterator ins = - insert(make_pair(SectionData::Identifier(*it, i, sectSort), SectionData(data))) - .first; - // note that it is guaranteed that the above insertion takes place, because this - // block is only entered - // if the sid does not exist inside this CellData object yet. - // if sorting is done by location, this will be unique anyways. - sidLookup.insert(make_pair(*it, ins)); - } else - // SID is already present in CellData. The new data item just needs to be inserted - segMatch->second.push_back(data); - } -} - -// typedef CellDataImpl UnsortedCellData; - -bool CellData::operator==(const CellData& d1) { - if (d1.size() != size()) - return false; - - const_iterator seg1, seg2; - - for (seg1 = d1.begin(), seg2 = begin(); seg2 != end(); seg1++, seg2++) { - if (seg1->first != seg2->first) - return false; - if (seg1->second != seg2->second) - return false; - } - - return true; -} - -ReadManager::ReadManager() {} - -ReadManager::ReadManager(const string& filePath) - : ifstream(filePath.c_str(), ios::in | ios::binary) { - exceptions(ios::badbit | ios::eofbit | ios::failbit); - // open(filePath); - readHeader(); -} - -void ReadManager::open(const string& filePath) { - exceptions(ios::badbit | ios::eofbit | ios::failbit); - - ifstream::open(filePath.c_str(), ios::in | ios::binary); - readHeader(); -} - -void ReadManager::readHeader() { - char readBuffer[HEADER_LENGTH + 1]; - - seekg(ios::beg); - - ifstream::read(readBuffer, HEADER_LENGTH); - - readBuffer[HEADER_LENGTH] = 0; // c string terminator - fileInfo = readBuffer; - - cellInfoOrigin = tellg(); - - isNative = fileInfo.sourceIsNative(); - - CellInfo firstCell = retrieveCellInfo(0); - - startLocation = firstCell.getDataLocation(); - mappingLocation = firstCell.getMappingLocation(); - - mappingSize_floats = fileInfo.getMappingSize() * fileInfo.getTotalNumberOfCompartments(); -} - -CellInfo ReadManager::makeNullInfo() { - char fakeBuffer[SIZE_CELL_INFO_LENGTH]; - - for (char *i = fakeBuffer, *bufferEnd = fakeBuffer + SIZE_CELL_INFO_LENGTH; i < bufferEnd; i++) - *i = 0; - - return CellInfo(fakeBuffer); -} - -const CellInfo ReadManager::_null_info = ReadManager::makeNullInfo(); - -CellInfo ReadManager::retrieveFindCell(CellID cellNum) { - int start = (cellNum > fileInfo.getNumberOfCells()) - ? fileInfo.getNumberOfCells() - 1 - : cellNum - 1; // in many cases cellNum will be same as cellIndex - CellInfo seek; - - bool oppositeSearch = false; - - int i = start; - do { - if (i >= (int) fileInfo.getNumberOfCells() || i < 0) { - if (oppositeSearch) - return makeNullInfo(); - else { - oppositeSearch = true; - i = start; - } - } - - seek = retrieveCellInfo(i); - - if ((int) seek.getCellNum() < i) - if (oppositeSearch) - i--; - else - i++; - else if (oppositeSearch) - i++; - else - i--; - - } while (seek.getCellNum() != cellNum); - - return seek; -} - -CellInfo ReadManager::retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode) { - FilePosition ppos = tellg(); - - seekgCellInfo(cellIndex); - - char readBuffer[SIZE_CELL_INFO_LENGTH]; - read(readBuffer, SIZE_CELL_INFO_LENGTH); - - CellInfo reqInfo(readBuffer, isNative, sortMode); - - seekg(ppos); - - return reqInfo; -} - -FrameInfo ReadManager::retrieveAllCellInfo(CellInfo::key sortMode) { - FilePosition ppos = tellg(); - - CellCount amountOfCells = fileInfo.getNumberOfCells(); - streamsize bytesToRead = SIZE_CELL_INFO_LENGTH * amountOfCells; - char* readBuffer = new char[bytesToRead]; - char* endOfBuffer = readBuffer + bytesToRead; - seekgCellInfo(0); - - read(readBuffer, bytesToRead); - - FrameInfo allCells(readBuffer, endOfBuffer, isNative, sortMode); - - delete[] readBuffer; - - seekg(ppos); - - return allCells; -} - -void ReadManager::readFrame(DataItem* dataBuffer) { - read(dataBuffer, fileInfo.getTotalNumberOfCompartments()); -} - -void ReadManager::readFrameMapping(MappingItem* buffer) { - FilePosition ppos = tellg(); - - seekgMappingStart(); - - read(buffer, mappingSize_floats); - - seekg(ppos); -} - -FrameParser::FrameOrganiser::FrameOrganiser() { - sectSort = SID; -} - -FrameParser::FrameOrganiser::FrameOrganiser(SortingKey segmentSorting) { - sectSort = segmentSorting; -} - -void FrameParser::FrameOrganiser::fillData(const FrameInfo& info, - const vector& mapping, - const DataItem* dataBuffer) { - FrameInfo::const_iterator pinf = info.begin(); - vector::const_iterator pmap = mapping.begin(); - - for (const DataItem* ptr = dataBuffer; pinf != info.end(); - ptr += pinf->getAmountOfCompartments(), pinf++, pmap++) { - // iteration through all gids - - iterator ins = insert(make_pair(*pinf, CellData(sectSort))).first; // insert CellData - // object for current - // gid - ins->second.fillData(ptr, *pmap); // feed new CellData object with mapping and data - } -} - -void FrameParser::FrameOrganiser::makeOrderedDataBuffer(DataItem* buffer) const { - for (const_iterator cd = begin(); cd != end(); cd++) // iterate through cells - for (CellData::const_iterator sd = cd->second.begin(); sd != cd->second.end(); - sd++) // iterate through segments - for (SectionData::const_iterator f = sd->second.begin(); f != sd->second.end(); - f++, buffer++) // iterate through compartments - *buffer = *f; -} - -FrameParser::CompartmentIndexing FrameParser::FrameOrganiser::makeOrderedOffsetReferences() const { - CompartmentIndexing gidV; - gidV.reserve(size()); - - FrameDataIndex position = 0; - - for (const_iterator gid = begin(); gid != end(); gid++) { // iterate through gids - gidV.push_back(vector()); - - // reserve memory for amount of segments - gidV.back().reserve((int) (gid->second.size() * 1.5)); - - MappingItem presid = 0; - for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); - position += sid->second.size(), sid++) { - // iterating through available sid's - while (presid++ != sid->first.getSID()) - gidV.back().push_back(0); // insert dummy variables for unavailable sids (gaps) - gidV.back().push_back(position); - } - } - - return gidV; -} - -FrameParser::CompartmentCounts FrameParser::FrameOrganiser::makeOrderedSegSizes() const { - CompartmentCounts gidV; - gidV.reserve(size()); - - for (const_iterator gid = begin(); gid != end(); gid++) { - gidV.push_back(vector()); - gidV.back().reserve((int) (gid->second.size() * 1.5)); - MappingItem presid = 0; - for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); sid++) { - while (presid++ != sid->first.getSID()) - gidV.back().push_back(0); // fill sid gaps with zero sizes - gidV.back().push_back(sid->second.size()); - } - } - - return gidV; -} - -FrameParser::FrameParser() { - readBuffer = NULL; - refArray = NULL; -} - -FrameParser::FrameParser(const string& file, bool sortData) - : ReadManager(file) { - if (fileInfo.getMappingSize() != 1) { - cerr << "Target file has incompatible mapping size for FrameParser. FrameParser only works " - "with mapping size 1." - << endl; - exit(1); - } - - CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; - - FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); - - ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); - offsets = ReadOptimiser(fileInfo.getTotalNumberOfCompartments()); // * sizeof(DataItem) ); - createReferences(cinfo, sortData); - - timestep = 0; // by default start pointing to timstep 0 - ReadManager::seekgFrame(timestep); -} - -FrameParser::FrameParser(const string& file, const vector& gidTarget, bool sortData) - : ReadManager(file) { - timestep = 0; - ReadOptimiser::setDefaultElementSize(sizeof(DataItem)); - retarget(gidTarget, sortData); -} - -FrameParser::~FrameParser() { - if (readBuffer) // readBuffer may have not been used if sorting was disabled, in which case - // it would have been set to NULL - delete[] readBuffer; - if (refArray) // same applies for refArray - delete[] refArray; -} - -void FrameParser::retarget(const vector& gidTarget, bool sortData) { - CellInfo::key cinfoSorting = (sortData) ? CellInfo::GID : CellInfo::LOCATION; - - FrameInfo cinfo = retrieveAllCellInfo(cinfoSorting); - cinfo.filterGIDs(gidTarget); - - offsets = ReadOptimiser(cinfo, getFrameSize_bytes(), getStartOffset()); - offsets.setElementSize(sizeof(DataItem)); - - createReferences(cinfo, sortData); - - seekg(getStartOffset() + offsets.getFirst()); - - operator=(timestep); -} - -void FrameParser::createReferences(const FrameInfo& cinfo, bool sortData) { - SortingKey sSorting = (sortData) ? SID : LOC; - - refArray = NULL; - readBuffer = NULL; - - vector allMapping; - - DataItem* tempBuffer = - new DataItem[offsets.getDataSize_elements()]; // allocate memory for readBuffer. This - // will atleast be needed for creating - // the FrameOrganiser - - readFrameMapping(tempBuffer); - - FrameInfo::const_iterator pinf = cinfo.begin(); - - for (DataItem *mpptr = tempBuffer, *endptr = mpptr + pinf->getAmountOfCompartments(); - pinf != cinfo.end(); - pinf++, mpptr = endptr, endptr += pinf->getAmountOfCompartments()) - allMapping.push_back(CompartmentMapping(mpptr, endptr)); - - if (sortData) - for (FrameDataIndex i = 0; i < offsets.getDataSize_elements(); i++) - tempBuffer[i] = i; // makes buffer of indexes { 0 , 1 ,2 , 3 , ... } - // else it doesnt really matter what FrameOrganiser is fed with. - - org = FrameOrganiser(sSorting); - - org.fillData(cinfo, allMapping, tempBuffer); // feed the FrameOrganiser with the data. - - // references = org.makeOrderedOffsetReferences(); - // segSizes = org.makeOrderedSegSizes(); - - if (sortData) { - readBuffer = tempBuffer; // keep allocated memory at tempBuffer - - // sortData is enabled -> create an array of pointers to readBuffer - // in which the pointers are sorted by GID and SID of the data the point to. - // these pointers are to be stored into ref array - org.makeOrderedDataBuffer(readBuffer); // rearranges indexes to correspond to gid and - // segment order - // the memory allocated for readBuffer is now being used for a different purpose: - // it contains the sorted indexes, which now need to be converted to pointers to - // readBuffer. - - refArray = new DataItem*[offsets.getDataSize_elements()]; // allocate memory for - // refArray (will contain - // pointers for sorting) - - for (float *pIndex = readBuffer, - **pPtr = refArray, - *pEnd = readBuffer + offsets.getDataSize_elements(); - pIndex != pEnd; - pIndex++, pPtr++) - *pPtr = &readBuffer[(FrameDataIndex) *pIndex]; // converts indexes to pointers and - // stores them into refArray - } else - // data sorting is disabled -> allocated memory at temp Buffer is no longer needed. - //(data will be read directly into the memory that user has provided) - delete[] tempBuffer; -} - -void FrameParser::bufferTransfer(DataItem* buffer) const { - DataItem* reqLocation = buffer; - - if (buffer == readBuffer) // source and destination is the same - trouble! - buffer = new DataItem[offsets.getDataSize_elements()]; // need to allocate different - // destination - - for (DataItem **sourcePtr = refArray, - *target = buffer, - *dEnd = buffer + offsets.getDataSize_elements(); - target != dEnd; - sourcePtr++, target++) - *target = **sourcePtr; - - if (reqLocation != buffer) { // buffer has not been transfered where requested! - // this can now be corrected. - for (DataItem *origin = buffer, - *destination = reqLocation, - *dEnd = buffer + offsets.getDataSize_elements(); - origin != dEnd; - origin++, destination++) - *destination = *origin; // copy data to requested location - delete[] buffer; // buffer was is only a temporary memory allocation - delete it. - } -} - -FrameParser& FrameParser::operator++() { - timestep++; - seekg(fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; -} - -FrameParser& FrameParser::operator++(int) { - return this->operator++(); -} - -FrameParser& FrameParser::operator--() { - timestep--; - seekg(-fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; -} - -FrameParser& FrameParser::operator--(int) { - return this->operator--(); -} - -FrameParser& FrameParser::operator+=(FrameDiff increment) { - timestep += increment; - seekg(increment * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; -} - -FrameParser& FrameParser::operator-=(FrameDiff decrement) { - timestep -= decrement; - seekg(-decrement * fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem), cur); - return *this; -} - -FrameParser& FrameParser::operator=(FrameIndex newTime) { - timestep = newTime; - seekgFrame(timestep); - seekg(offsets.getFirst(), cur); - return *this; -} - -Time FrameParser::getTime() const { - return fileInfo.getStartTime() + timestep * fileInfo.getTimeStepSize(); -} - -void FrameParser::readFrame(DataItem* buffer) { - DataItem* readPtr; - - if (readBuffer) // readBuffer will be NULL if sorting is disabled - readPtr = readBuffer; // sorting enabled -> read into readBuffer and transfer data in - // order later - else - readPtr = buffer; // sorting disabled -> read directly into buffer - - // offsets contains alternating values of data sizes to be read and skipped. - for (ReadOptimiser::iterator i = offsets.begin(); i != offsets.end(); i++) { - read(readPtr, *i); - readPtr += *i; - i++; - - // seems to be confusion as to whether offsets should contain bytelength or element - // count. I have it contain element count for now - // i now points to an offset that indicates amount of data to be skipped - if (i == offsets.end()) // no more offsets -> i'm done reading - break; - - if (*i) - seekg(*i, ios::cur); // non-zero offset -> perfom the skip - } - - if (refArray) // refArray will be NULL if sorting is disabled - bufferTransfer(buffer); // sorting is enabled. Transfer data from readBuffer to buffer - // such that data is in order -} - -void FrameParser::readFrameMapping(MappingItem* buffer) { - // readFrameMapping should not return without the filepointer being in the same state as - // before it was called. - FilePosition before = tellg(); // make a backup for current filepointer - - seekgMappingStart(); // now move file pointer to mapping location - - size_t prevElement = offsets.getElementSize(); // backup element size. - offsets.setElementSize(sizeof(MappingItem)); // setup offsets to be reading MappingItems - // instead of whatever it was reading before. - - seekg(offsets.getFirst(), ios::cur); // and advance to first cell - readFrame(buffer); // read the data - this function will also alter the filepointer and - // timestep - - offsets.setElementSize(prevElement); // restore element size. - - seekg(before); // restore filepointer -} - -void FrameParser::readFrameData(DataItem* buffer) { - readFrame(buffer); - timestep++; // the filepointer has been incremented to the next timestep as data has been - // read. -} - -size_t FrameParser::ReadOptimiser::defaultElementSize = 1; - -FrameParser::ReadOptimiser::ReadOptimiser() { - elementSize = 0; - firstOffset = 0; - dataSize_elements = 0; -} - -FrameParser::ReadOptimiser::ReadOptimiser(std::streamsize frameSize_bytes) { - elementSize = defaultElementSize; - firstOffset = 0; - dataSize_elements = frameSize_bytes; // * elementSize; - push_back(dataSize_elements); -} - -FrameParser::ReadOptimiser::ReadOptimiser(const FrameInfo& cellsToRead, - std::streamsize frameSize_bytes, - FilePosition startLocation) { - elementSize = defaultElementSize; - // collect data edge points - multiset dataEdges; - - dataEdges.insert(startLocation); - - for (FrameInfo::const_iterator i = cellsToRead.begin(); i != cellsToRead.end(); i++) { - dataEdges.insert(i->getDataLocation()); // starting point - dataEdges.insert(i->getDataLocation() + - i->getAmountOfCompartments() * elementSize); // end point - } - - dataEdges.insert(startLocation + (FileOffset) frameSize_bytes); - - /*create a vector that contains the offsets. - * The first offset represents the first bunch of data to be read. - * The second offset then represents the amound of data to be skipped after the first read. - * and then the next offset is again the amount of data to be read. - */ - - // compute the offsets differentiating the dataEdges - list offsets; - for (multiset::iterator currentPos = dataEdges.begin(), prevPos = currentPos++; - currentPos != dataEdges.end(); - prevPos = currentPos, currentPos++) - offsets.push_back(*currentPos - *prevPos); - - // the last offsets should move the file pointer beyond the frame to the start location on - // the next frame - // therefore the first and last offsets are special -> take them out - - firstOffset = offsets.front(); - FileOffset lastOffset = offsets.back(); - - offsets.pop_front(); // remove first offset from list - offsets.pop_back(); // remove last offset from list - - // now the first an last offset are read dimensions. This guarantees that a skip has always - // a precessor and successor - // this is convenient in the next step, as a skip is always on an even location aswell as - // offsets.end() - - list::iterator it; - bool toggleOnRead; - - dataSize_elements = 0; - for (toggleOnRead = true, it = offsets.begin(); it != offsets.end(); - toggleOnRead = !toggleOnRead, it++) { - if (toggleOnRead) { - dataSize_elements += (*it /= elementSize); - // handle read - // read specifications should be in units of floats, but skips should be in units of - // bytes. - // also size of data to be read per frame needs to established - } else - // handle skip - There should be no zero offset inbetween - if (*it == 0) { - // remove this zero skip by combining prev. and next read. - list::iterator prevRead = it; - list::iterator nextRead = it; - prevRead--; - nextRead++; - - dataSize_elements += (*nextRead /= elementSize); // next read was still in units of - // bytes - *prevRead += *nextRead; // combine nextRead with prevRead - offsets.erase(it); - offsets.erase(nextRead); - it = prevRead; // point to prevRead now - toggleOnRead = true; // and mark that toggle flag that current iteration deal with - // read. - } - } - - // now optimise access performance by converting the offsets list to a vector. - - vector voffsets(offsets.begin(), offsets.end()); - swap(voffsets); - - // compute the finalSkip, which brings the filePointer back to the starting location, but of - // the nextFrame - FileOffset finalSkip = lastOffset + firstOffset; - - if (finalSkip) - push_back(finalSkip); -} - -void FrameParser::ReadOptimiser::updateOffsets(size_t newSize) { - dataSize_elements = newSize * dataSize_elements / elementSize; - for (iterator i = begin(); i != end(); i++) { // iterate through all read offsets - // first reduce the read offsets to byte numbers, then convert it to the new element - // size - *i = newSize * (*i) / elementSize; - i++; - if (i == end()) - break; - } -} -} // namespace bbpReader diff --git a/tools/converter/binary_reader/binary_positions.h b/tools/converter/binary_reader/binary_positions.h new file mode 100644 index 0000000..ecdf0b6 --- /dev/null +++ b/tools/converter/binary_reader/binary_positions.h @@ -0,0 +1,62 @@ +// FixedPositions // Don't change so we can have backward compatibility + +// If identifier read at position 0 matches ARCHITECTURE_IDENTIFIER, then the file was writting from +// native architecture +#define SONATA_REPORT_ARCHITECTURE_IDENTIFIER 1.001 + +// Double: +#define SONATA_REPORT_IDENTIFIER_POSITION 0 +// Int: +#define SONATA_REPORT_HEADER_SIZE_POSITION SONATA_REPORT_IDENTIFIER_POSITION + sizeof(double) +// String: +#define SONATA_REPORT_LIBRARY_VERSION_POSITION 16 +// String: +#define SONATA_REPORT_SIMULATOR_VERSION_POSITION 32 +// Int: +#define SONATA_REPORT_TOTAL_NUMBER_OF_CELLS_POSITION 48 +// Int: +#define SONATA_REPORT_TOTAL_NUMBER_OF_COMPARTMENTS_POSITION 52 +// Int: +#define SONATA_REPORT_NUMBER_OF_STEPS_POSITION 64 +// Double: +#define SONATA_REPORT_TIME_START_POSITION 72 +// Double: +#define SONATA_REPORT_TIME_END_POSITION 80 +// Double: +#define SONATA_REPORT_DT_TIME_POSITION 88 +// String: +#define SONATA_REPORT_D_UNIT_POSITION 96 +// String: +#define SONATA_REPORT_T_UNIT_POSITION 112 +// Int: +#define SONATA_REPORT_MAPPING_SIZE_POSITION 128 +// String: +#define SONATA_REPORT_MAPPING_NAME_POSITION 144 +// Int: +#define SONATA_REPORT_EXTRA_MAPPING_SIZE_POSITION 160 +// String: +#define SONATA_REPORT_EXTRA_MAPPING_NAME_POSITION 176 +// String: +#define SONATA_REPORT_TARGET_NAME_POSITION 192 + +// Length definition +// int: +#define SONATA_REPORT_HEADER_LENGTH 1024 +// int: +#define SONATA_REPORT_SIZE_CELL_INFO_LENGTH 64 +// int: +#define SONATA_REPORT_SIZE_STRING 16 + +// Inside CellInfo +// Int: +#define SONATA_REPORT_NUMBER_OF_CELL_POSITION 0 +// Int: +#define SONATA_REPORT_NUMBER_OF_COMPARTMENTS_POSITION 8 +// OffSet: +#define SONATA_REPORT_DATA_INFO_POSITION 16 +// OffSet: +#define SONATA_REPORT_EXTRA_MAPPING_INFO_POSITION 24 +// OffSet: +#define SONATA_REPORT_MAPPING_INFO_POSITION 32 + +// order of the cell is a simple string order diff --git a/tools/converter/binary_reader/binary_reader.cpp b/tools/converter/binary_reader/binary_reader.cpp new file mode 100644 index 0000000..3a9253f --- /dev/null +++ b/tools/converter/binary_reader/binary_reader.cpp @@ -0,0 +1,883 @@ +#include "binary_reader.h" +#include +#include + +using namespace std; + +namespace bbp { +namespace binary_reader { + +Header::Header(char* read_buffer) { +#define fetch(TYPE, POSITION) *((TYPE*) (read_buffer + POSITION)) +#define fetchS(POSITION) string((char*) (read_buffer + POSITION)) + + double read_identifier = fetch(double, SONATA_REPORT_IDENTIFIER_POSITION); + header_size = fetch(int32_t, SONATA_REPORT_HEADER_SIZE_POSITION); + amount_of_cells = fetch(int32_t, SONATA_REPORT_TOTAL_NUMBER_OF_CELLS_POSITION); + /** \bug 32 bit is consistent with the binary format specification of the + header, however + */ + uint32_t compartments_per_frame = fetch(int32_t, + SONATA_REPORT_TOTAL_NUMBER_OF_COMPARTMENTS_POSITION); + library_version = fetchS(SONATA_REPORT_LIBRARY_VERSION_POSITION); + simulator_version = fetchS(SONATA_REPORT_SIMULATOR_VERSION_POSITION); + number_of_steps = fetch(int32_t, SONATA_REPORT_NUMBER_OF_STEPS_POSITION); + start_time = fetch(double, SONATA_REPORT_TIME_START_POSITION); + end_time = fetch(double, SONATA_REPORT_TIME_END_POSITION); + dt = fetch(double, SONATA_REPORT_DT_TIME_POSITION); + data_unit = fetchS(SONATA_REPORT_D_UNIT_POSITION); + time_unit = fetchS(SONATA_REPORT_T_UNIT_POSITION); + mapping_size = fetch(int32_t, SONATA_REPORT_MAPPING_SIZE_POSITION); + mapping_name = fetchS(SONATA_REPORT_MAPPING_NAME_POSITION); + extra_mapping_size = fetch(int32_t, SONATA_REPORT_EXTRA_MAPPING_SIZE_POSITION); + extra_mapping_name = fetchS(SONATA_REPORT_EXTRA_MAPPING_NAME_POSITION); + target_name = fetchS(SONATA_REPORT_TARGET_NAME_POSITION); + +#undef fetchS +#undef fetch + + is_native = read_identifier == SONATA_REPORT_ARCHITECTURE_IDENTIFIER; + + if (!is_native) { + // check if data can be converted: + DOUBLESWAP(read_identifier); + if (read_identifier != SONATA_REPORT_ARCHITECTURE_IDENTIFIER) { + cerr << "File is corrupt or originated from an unknown architecture." << endl; + exit(1); + } + + // if data was stored on a machine with different architecture, + // then the data needs to be converted to be compatible with the reading machine + FIX_INT(header_size); + FIX_INT(amount_of_cells); + FIX_INT(compartments_per_frame); + FIX_INT(number_of_steps); + DOUBLESWAP(start_time); + DOUBLESWAP(end_time); + DOUBLESWAP(dt); + FIX_INT(mapping_size); + FIX_INT(extra_mapping_size); + } + + total_amount_of_compartments = compartments_per_frame; +} + +bool Header::operator==(const Header& h1) const { +#define _CHECK(field) \ + if (h1.field != field) \ + return false + // compare all fields other than is_native + _CHECK(header_size); + _CHECK(library_version); + _CHECK(simulator_version); + _CHECK(amount_of_cells); + _CHECK(total_amount_of_compartments); + _CHECK(number_of_steps); + _CHECK(start_time); + _CHECK(end_time); + _CHECK(dt); + _CHECK(data_unit); + _CHECK(time_unit); + _CHECK(mapping_size); + _CHECK(mapping_name); + _CHECK(extra_mapping_name); + _CHECK(extra_mapping_size); + _CHECK(target_name); + + // else if nothing returned false: + return true; +#undef _CHECK +} + +CellInfo::CellInfo() { + cell_num = 0; + master = &cell_num; + amount_of_compartments = 0; + data_location = 0; + extra_mapping_location = 0; + mapping_location = 0; +} + +CellInfo::CellInfo(char* read_buffer, bool is_native, key cmp) { +#define fetch(TYPE, POSITION) *((TYPE*) (read_buffer + POSITION)) + cell_num = fetch(int32_t, SONATA_REPORT_NUMBER_OF_CELL_POSITION); + amount_of_compartments = fetch(int32_t, SONATA_REPORT_NUMBER_OF_COMPARTMENTS_POSITION); + + data_location = fetch(uint64_t, SONATA_REPORT_DATA_INFO_POSITION); + extra_mapping_location = fetch(uint64_t, SONATA_REPORT_EXTRA_MAPPING_INFO_POSITION); + mapping_location = fetch(uint64_t, SONATA_REPORT_MAPPING_INFO_POSITION); + +#undef fetch + + if (!is_native) { + FIX_INT(cell_num); + FIX_INT(amount_of_compartments); + + DOUBLESWAP(data_location); + DOUBLESWAP(extra_mapping_location); + DOUBLESWAP(mapping_location); + } + + if (cmp == GID) + master = &cell_num; + else + master = &data_location; +} + +bool CellInfo::operator==(const CellInfo& c1) const { +#define _CHECK(field) \ + if (c1.field != field) \ + return false + _CHECK(cell_num); + _CHECK(amount_of_compartments); + _CHECK(data_location); + _CHECK(extra_mapping_location); + _CHECK(mapping_location); + + return true; +#undef _CHECK +} + +bool CellInfo::operator!=(const CellInfo& c1) const { + return !(*this == c1); +} + +#define COMPARE_(op) \ + (master == &cell_num) ? cell_num op c.cell_num : data_location op c.data_location + +#define DEFINE_CMP_OP(op) \ + bool CellInfo::operator op(const CellInfo& c) const { \ + return COMPARE_(op); \ + } + +DEFINE_CMP_OP(<) +DEFINE_CMP_OP(<=) +DEFINE_CMP_OP(>) +DEFINE_CMP_OP(>=) + +#undef DEFINE_CMP_OP +#undef COMPARE_ + +bool CellInfo::compare_by_offset(const CellInfo& a, const CellInfo& b) { + return a.data_location < b.data_location; +} + +FrameInfo::FrameInfo(char* buffer, char* end_of_buffer, bool is_native, CellInfo::key sort_mode) { + // create a cellinfo and then insert into itself (rem, this class inherits from set). The + // resulting set iterator is then put into + // a map member for associative lookup gid:location + for (char* buffer_loc = buffer; buffer_loc < end_of_buffer; + buffer_loc += SONATA_REPORT_SIZE_CELL_INFO_LENGTH) { + iterator ins = insert(CellInfo(buffer_loc, is_native, sort_mode)).first; + gid_lookup.insert(make_pair(ins->get_cell_num(), ins)); + } +} + +FrameInfo::FrameInfo(const FrameInfo& rhs) + : baseTypes::FrameInfo() { + *this = rhs; +} + +FrameInfo& FrameInfo::operator=(const FrameInfo& rhs) { + insert(rhs.begin(), rhs.end()); + for (iterator cell_item = begin(); cell_item != end(); ++cell_item) { + gid_lookup.insert(make_pair(cell_item->get_cell_num(), cell_item)); + } + + return *this; +} + +FrameInfo& FrameInfo::filter_gids(const vector& gids) { + Lookup new_gid_lookup; + + for (vector::const_iterator gid = gids.begin(); gid != gids.end(); gid++) { + Lookup::iterator it = gid_lookup.find(*gid); + if (it != gid_lookup.end()) { + new_gid_lookup.insert(*it); + gid_lookup.erase(it); + } else + throw string("GID doesn't exist in file."); // TODO: replace by custom std::exception + } + + // gid_lookup contains the gids that need to be removed and the new_gid_lookup contains the + // gids that remain. + + for (Lookup::iterator i = gid_lookup.begin(); i != gid_lookup.end(); i++) + erase(i->second); // delete the unwanted gids. + + gid_lookup.swap(new_gid_lookup); + + return *this; +} + +CompartmentMapping::CompartmentMapping() + : Base() {} + +CompartmentMapping::CompartmentMapping(MappingItem* start, MappingItem* _end) + : Base(start, _end) {} + +CompartmentMapping::CompartmentMapping(MappingItem* buffer, SizeType buf_size) + : Base(buffer, buffer + buf_size) {} + +SectionData::SectionData() + : Base() {} + +SectionData::SectionData(DataItem first_data) { + push_back(first_data); +} + +SectionData::SectionData(DataItem* start, DataItem* _end) + : Base(start, _end) {} + +bool SectionData::operator==(const SectionData& s1) { + if (s1.size() != size()) + return false; + + const_iterator dat1, dat2; + for (dat1 = s1.begin(), dat2 = begin(); dat2 != end(); dat1++, dat2++) + if (*dat1 != *dat2) + return false; + + return true; +} + +bool SectionData::operator!=(const SectionData& s1) { + return !(*this == s1); +} + +bool SectionData::compare_by_original_location(const Identifier& c1, const Identifier& c2) { + return c1.get_location() < c2.get_location(); +} + +CellData::CellData() { + sect_sort = SID; +} + +CellData::CellData(SortingKey sorting) { + sect_sort = sorting; +} + +CellData::CellData(const DataItem* data_buffer, const CompartmentMapping& mapping) { + sect_sort = SID; + fill_data(data_buffer, mapping); +} + +void CellData::fill_data(const DataItem* data_buffer, const CompartmentMapping& mapping) { + CompartmentMapping::const_iterator it; + SectionIndex i; + for (it = mapping.begin(), i = 0; it != mapping.end(); it++, data_buffer++, i++) { + // iteration through all compartments inside cell + DataItem data = *data_buffer; + + iterator seg_match = find(*it); // find corresponding segment + + if (seg_match == end()) { + // SID was not found. + // this means it wasnt created yet. + // therefore create a new one and insert first data item into it: + iterator ins = + insert(make_pair(SectionData::Identifier(*it, i, sect_sort), SectionData(data))) + .first; + // note that it is guaranteed that the above insertion takes place, because this + // block is only entered + // if the sid does not exist inside this CellData object yet. + // if sorting is done by location, this will be unique anyways. + sid_lookup.insert(make_pair(*it, ins)); + } else + // SID is already present in CellData. The new data item just needs to be inserted + seg_match->second.push_back(data); + } +} + +bool CellData::operator==(const CellData& d1) { + if (d1.size() != size()) + return false; + + const_iterator seg1, seg2; + + for (seg1 = d1.begin(), seg2 = begin(); seg2 != end(); seg1++, seg2++) { + if (seg1->first != seg2->first) + return false; + if (seg1->second != seg2->second) + return false; + } + + return true; +} + +ReadManager::ReadManager() {} + +ReadManager::ReadManager(const string& file_path) + : ifstream(file_path.c_str(), ios::in | ios::binary) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); + read_header(); +} + +void ReadManager::open(const string& file_path) { + exceptions(ios::badbit | ios::eofbit | ios::failbit); + + ifstream::open(file_path.c_str(), ios::in | ios::binary); + read_header(); +} + +void ReadManager::read_header() { + char read_buffer[SONATA_REPORT_HEADER_LENGTH + 1]; + + seekg(ios::beg); + + ifstream::read(read_buffer, SONATA_REPORT_HEADER_LENGTH); + + read_buffer[SONATA_REPORT_HEADER_LENGTH] = 0; // c string terminator + file_info = read_buffer; + + cell_info_origin = tellg(); + + is_native = file_info.source_is_native(); + + CellInfo first_cell = retrieve_cell_info(0); + + start_location = first_cell.get_data_location(); + mapping_location = first_cell.get_mapping_location(); + + mapping_size_floats = file_info.get_mapping_size() * + file_info.get_total_number_of_compartments(); +} + +CellInfo ReadManager::make_null_info() { + char fake_buffer[SONATA_REPORT_SIZE_CELL_INFO_LENGTH]; + + for (char *i = fake_buffer, *buffer_end = fake_buffer + SONATA_REPORT_SIZE_CELL_INFO_LENGTH; + i < buffer_end; + i++) + *i = 0; + + return CellInfo(fake_buffer); +} + +const CellInfo ReadManager::_null_info = ReadManager::make_null_info(); + +CellInfo ReadManager::retrieve_find_cell(CellID cell_num) { + int start = (cell_num > file_info.get_number_of_cells()) + ? file_info.get_number_of_cells() - 1 + : cell_num - 1; // in many cases cellNum will be same as cellIndex + CellInfo seek; + + bool opposite_search = false; + + int i = start; + do { + if (i >= (int) file_info.get_number_of_cells() || i < 0) { + if (opposite_search) + return make_null_info(); + else { + opposite_search = true; + i = start; + } + } + + seek = retrieve_cell_info(i); + + if ((int) seek.get_cell_num() < i) + if (opposite_search) + i--; + else + i++; + else if (opposite_search) + i++; + else + i--; + + } while (seek.get_cell_num() != cell_num); + + return seek; +} + +CellInfo ReadManager::retrieve_cell_info(CellIndex cell_index, CellInfo::key sort_mode) { + FilePosition ppos = tellg(); + + seekg_cell_info(cell_index); + + char read_buffer[SONATA_REPORT_SIZE_CELL_INFO_LENGTH]; + read(read_buffer, SONATA_REPORT_SIZE_CELL_INFO_LENGTH); + + CellInfo req_info(read_buffer, is_native, sort_mode); + + seekg(ppos); + + return req_info; +} + +FrameInfo ReadManager::retrieve_all_cell_info(CellInfo::key sort_mode) { + FilePosition ppos = tellg(); + + CellCount amount_of_cells = file_info.get_number_of_cells(); + streamsize bytes_to_read = SONATA_REPORT_SIZE_CELL_INFO_LENGTH * amount_of_cells; + char* read_buffer = new char[bytes_to_read]; + char* end_of_buffer = read_buffer + bytes_to_read; + seekg_cell_info(0); + + read(read_buffer, bytes_to_read); + + FrameInfo all_cells(read_buffer, end_of_buffer, is_native, sort_mode); + + delete[] read_buffer; + + seekg(ppos); + + return all_cells; +} + +void ReadManager::read_frame(DataItem* data_buffer) { + read(data_buffer, file_info.get_total_number_of_compartments()); +} + +void ReadManager::read_frame_mapping(MappingItem* buffer) { + FilePosition ppos = tellg(); + + seekg_mapping_start(); + + read(buffer, mapping_size_floats); + + seekg(ppos); +} + +FrameParser::FrameOrganiser::FrameOrganiser() { + sect_sort = SID; +} + +FrameParser::FrameOrganiser::FrameOrganiser(SortingKey segment_sorting) { + sect_sort = segment_sorting; +} + +void FrameParser::FrameOrganiser::fill_data(const FrameInfo& info, + const vector& mapping, + const DataItem* data_buffer) { + FrameInfo::const_iterator pinf = info.begin(); + vector::const_iterator pmap = mapping.begin(); + + for (const DataItem* ptr = data_buffer; pinf != info.end(); + ptr += pinf->get_amount_of_compartments(), pinf++, pmap++) { + // iteration through all gids + + iterator ins = insert(make_pair(*pinf, CellData(sect_sort))).first; // insert CellData + // object for current + // gid + ins->second.fill_data(ptr, *pmap); // feed new CellData object with mapping and data + } +} + +void FrameParser::FrameOrganiser::make_ordered_data_buffer(DataItem* buffer) const { + for (const_iterator cd = begin(); cd != end(); cd++) // iterate through cells + for (CellData::const_iterator sd = cd->second.begin(); sd != cd->second.end(); + sd++) // iterate through segments + for (SectionData::const_iterator f = sd->second.begin(); f != sd->second.end(); + f++, buffer++) // iterate through compartments + *buffer = *f; +} + +FrameParser::CompartmentIndexing FrameParser::FrameOrganiser::make_ordered_offset_references() + const { + CompartmentIndexing gid_v; + gid_v.reserve(size()); + + FrameDataIndex position = 0; + + for (const_iterator gid = begin(); gid != end(); gid++) { // iterate through gids + gid_v.push_back(vector()); + + // reserve memory for amount of segments + gid_v.back().reserve((int) (gid->second.size() * 1.5)); + + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); + position += sid->second.size(), sid++) { + // iterating through available sid's + while (presid++ != sid->first.get_sid()) + gid_v.back().push_back(0); // insert dummy variables for unavailable sids (gaps) + gid_v.back().push_back(position); + } + } + + return gid_v; +} + +FrameParser::CompartmentCounts FrameParser::FrameOrganiser::make_ordered_seg_sizes() const { + CompartmentCounts gid_v; + gid_v.reserve(size()); + + for (const_iterator gid = begin(); gid != end(); gid++) { + gid_v.push_back(vector()); + gid_v.back().reserve((int) (gid->second.size() * 1.5)); + MappingItem presid = 0; + for (CellData::const_iterator sid = gid->second.begin(); sid != gid->second.end(); sid++) { + while (presid++ != sid->first.get_sid()) + gid_v.back().push_back(0); // fill sid gaps with zero sizes + gid_v.back().push_back(sid->second.size()); + } + } + + return gid_v; +} + +FrameParser::FrameParser() { + read_buffer = NULL; + ref_array = NULL; +} + +FrameParser::FrameParser(const string& file, bool sort_data) + : ReadManager(file) { + if (file_info.get_mapping_size() != 1) { + cerr << "Target file has incompatible mapping size for FrameParser. FrameParser only works " + "with mapping size 1." + << endl; + exit(1); + } + + CellInfo::key cinfo_sorting = (sort_data) ? CellInfo::GID : CellInfo::LOCATION; + + FrameInfo cinfo = retrieve_all_cell_info(cinfo_sorting); + + ReadOptimiser::set_default_element_size(sizeof(DataItem)); + offsets = ReadOptimiser(file_info.get_total_number_of_compartments()); + create_references(cinfo, sort_data); + + timestep = 0; // by default start pointing to timstep 0 + ReadManager::seekg_frame(timestep); +} + +FrameParser::FrameParser(const string& file, const vector& gid_target, bool sort_data) + : ReadManager(file) { + timestep = 0; + ReadOptimiser::set_default_element_size(sizeof(DataItem)); + retarget(gid_target, sort_data); +} + +FrameParser::~FrameParser() { + if (read_buffer) // readBuffer may have not been used if sorting was disabled, in which case + // it would have been set to NULL + delete[] read_buffer; + if (ref_array) // same applies for refArray + delete[] ref_array; +} + +void FrameParser::retarget(const vector& gid_target, bool sort_data) { + CellInfo::key cinfo_sorting = (sort_data) ? CellInfo::GID : CellInfo::LOCATION; + + FrameInfo cinfo = retrieve_all_cell_info(cinfo_sorting); + cinfo.filter_gids(gid_target); + + offsets = ReadOptimiser(cinfo, get_frame_size_bytes(), get_start_offset()); + offsets.set_element_size(sizeof(DataItem)); + + create_references(cinfo, sort_data); + + seekg(get_start_offset() + offsets.get_first()); + + operator=(timestep); +} + +void FrameParser::create_references(const FrameInfo& cinfo, bool sort_data) { + SortingKey s_sorting = (sort_data) ? SID : LOC; + + ref_array = NULL; + read_buffer = NULL; + + vector all_mapping; + + DataItem* temp_buffer = + new DataItem[offsets.get_data_size_elements()]; // allocate memory for readBuffer. This + // will atleast be needed for creating + // the FrameOrganiser + + read_frame_mapping(temp_buffer); + + FrameInfo::const_iterator pinf = cinfo.begin(); + + for (DataItem *mpptr = temp_buffer, *endptr = mpptr + pinf->get_amount_of_compartments(); + pinf != cinfo.end(); + pinf++, mpptr = endptr, endptr += pinf->get_amount_of_compartments()) + all_mapping.push_back(CompartmentMapping(mpptr, endptr)); + + if (sort_data) + for (FrameDataIndex i = 0; i < offsets.get_data_size_elements(); i++) + temp_buffer[i] = i; // makes buffer of indexes { 0 , 1 ,2 , 3 , ... } + // else it doesnt really matter what FrameOrganiser is fed with. + + org = FrameOrganiser(s_sorting); + + org.fill_data(cinfo, all_mapping, temp_buffer); // feed the FrameOrganiser with the data. + + if (sort_data) { + read_buffer = temp_buffer; // keep allocated memory at tempBuffer + + // sort_data is enabled -> create an array of pointers to readBuffer + // in which the pointers are sorted by GID and SID of the data the point to. + // these pointers are to be stored into ref array + org.make_ordered_data_buffer(read_buffer); // rearranges indexes to correspond to gid and + // segment order + // the memory allocated for readBuffer is now being used for a different purpose: + // it contains the sorted indexes, which now need to be converted to pointers to + // readBuffer. + + ref_array = new DataItem*[offsets.get_data_size_elements()]; // allocate memory for + // refArray (will contain + // pointers for sorting) + + for (float *p_index = read_buffer, + **p_ptr = ref_array, + *p_end = read_buffer + offsets.get_data_size_elements(); + p_index != p_end; + p_index++, p_ptr++) + *p_ptr = &read_buffer[(FrameDataIndex) *p_index]; // converts indexes to pointers and + // stores them into refArray + } else + // data sorting is disabled -> allocated memory at temp Buffer is no longer needed. + //(data will be read directly into the memory that user has provided) + delete[] temp_buffer; +} + +void FrameParser::buffer_transfer(DataItem* buffer) const { + DataItem* req_location = buffer; + + if (buffer == read_buffer) // source and destination is the same - trouble! + buffer = new DataItem[offsets.get_data_size_elements()]; // need to allocate different + // destination + + for (DataItem **source_ptr = ref_array, + *target = buffer, + *d_end = buffer + offsets.get_data_size_elements(); + target != d_end; + source_ptr++, target++) + *target = **source_ptr; + + if (req_location != buffer) { // buffer has not been transfered where requested! + // this can now be corrected. + for (DataItem *origin = buffer, + *destination = req_location, + *d_end = buffer + offsets.get_data_size_elements(); + origin != d_end; + origin++, destination++) + *destination = *origin; // copy data to requested location + delete[] buffer; // buffer was is only a temporary memory allocation - delete it. + } +} + +FrameParser& FrameParser::operator++() { + timestep++; + seekg(file_info.get_total_number_of_compartments() * sizeof(DataItem), cur); + return *this; +} + +FrameParser& FrameParser::operator++(int) { + return this->operator++(); +} + +FrameParser& FrameParser::operator--() { + timestep--; + seekg(-file_info.get_total_number_of_compartments() * sizeof(DataItem), cur); + return *this; +} + +FrameParser& FrameParser::operator--(int) { + return this->operator--(); +} + +FrameParser& FrameParser::operator+=(FrameDiff increment) { + timestep += increment; + seekg(increment * file_info.get_total_number_of_compartments() * sizeof(DataItem), cur); + return *this; +} + +FrameParser& FrameParser::operator-=(FrameDiff decrement) { + timestep -= decrement; + seekg(-decrement * file_info.get_total_number_of_compartments() * sizeof(DataItem), cur); + return *this; +} + +FrameParser& FrameParser::operator=(FrameIndex new_time) { + timestep = new_time; + seekg_frame(timestep); + seekg(offsets.get_first(), cur); + return *this; +} + +Time FrameParser::get_time() const { + return file_info.get_start_time() + timestep * file_info.get_time_step_size(); +} + +void FrameParser::read_frame(DataItem* buffer) { + DataItem* read_ptr; + + if (read_buffer) // readBuffer will be NULL if sorting is disabled + read_ptr = read_buffer; // sorting enabled -> read into readBuffer and transfer data in + // order later + else + read_ptr = buffer; // sorting disabled -> read directly into buffer + + // offsets contains alternating values of data sizes to be read and skipped. + for (ReadOptimiser::iterator i = offsets.begin(); i != offsets.end(); i++) { + read(read_ptr, *i); + read_ptr += *i; + i++; + + // seems to be confusion as to whether offsets should contain bytelength or element + // count. I have it contain element count for now + // i now points to an offset that indicates amount of data to be skipped + if (i == offsets.end()) // no more offsets -> i'm done reading + break; + + if (*i) + seekg(*i, ios::cur); // non-zero offset -> perfom the skip + } + + if (ref_array) // refArray will be NULL if sorting is disabled + buffer_transfer(buffer); // sorting is enabled. Transfer data from readBuffer to buffer + // such that data is in order +} + +void FrameParser::read_frame_mapping(MappingItem* buffer) { + // readFrameMapping should not return without the filepointer being in the same state as + // before it was called. + FilePosition before = tellg(); // make a backup for current filepointer + + seekg_mapping_start(); // now move file pointer to mapping location + + size_t prev_element = offsets.get_element_size(); // backup element size. + offsets.set_element_size(sizeof(MappingItem)); // setup offsets to be reading MappingItems + // instead of whatever it was reading before. + + seekg(offsets.get_first(), ios::cur); // and advance to first cell + read_frame(buffer); // read the data - this function will also alter the filepointer and + // timestep + + offsets.set_element_size(prev_element); // restore element size. + + seekg(before); // restore filepointer +} + +void FrameParser::read_frame_data(DataItem* buffer) { + read_frame(buffer); + timestep++; // the filepointer has been incremented to the next timestep as data has been + // read. +} + +size_t FrameParser::ReadOptimiser::default_element_size = 1; + +FrameParser::ReadOptimiser::ReadOptimiser() { + element_size = 0; + first_offset = 0; + data_size_elements = 0; +} + +FrameParser::ReadOptimiser::ReadOptimiser(std::streamsize frame_size_bytes) { + element_size = default_element_size; + first_offset = 0; + data_size_elements = frame_size_bytes; // * element_size; + push_back(data_size_elements); +} + +FrameParser::ReadOptimiser::ReadOptimiser(const FrameInfo& cells_to_read, + std::streamsize frame_size_bytes, + FilePosition start_location) { + element_size = default_element_size; + // collect data edge points + multiset data_edges; + + data_edges.insert(start_location); + + for (FrameInfo::const_iterator i = cells_to_read.begin(); i != cells_to_read.end(); i++) { + data_edges.insert(i->get_data_location()); // starting point + data_edges.insert(i->get_data_location() + + i->get_amount_of_compartments() * element_size); // end point + } + + data_edges.insert(start_location + (FileOffset) frame_size_bytes); + + /*create a vector that contains the offsets. + * The first offset represents the first bunch of data to be read. + * The second offset then represents the amound of data to be skipped after the first read. + * and then the next offset is again the amount of data to be read. + */ + + // compute the offsets differentiating the dataEdges + list offsets; + for (multiset::iterator current_pos = data_edges.begin(), + prev_pos = current_pos++; + current_pos != data_edges.end(); + prev_pos = current_pos, current_pos++) + offsets.push_back(*current_pos - *prev_pos); + + // the last offsets should move the file pointer beyond the frame to the start location on + // the next frame + // therefore the first and last offsets are special -> take them out + + first_offset = offsets.front(); + FileOffset last_offset = offsets.back(); + + offsets.pop_front(); // remove first offset from list + offsets.pop_back(); // remove last offset from list + + // now the first an last offset are read dimensions. This guarantees that a skip has always + // a precessor and successor + // this is convenient in the next step, as a skip is always on an even location aswell as + // offsets.end() + + list::iterator it; + bool toggle_on_read; + + data_size_elements = 0; + for (toggle_on_read = true, it = offsets.begin(); it != offsets.end(); + toggle_on_read = !toggle_on_read, it++) { + if (toggle_on_read) { + data_size_elements += (*it /= element_size); + // handle read + // read specifications should be in units of floats, but skips should be in units of + // bytes. + // also size of data to be read per frame needs to established + } else + // handle skip - There should be no zero offset inbetween + if (*it == 0) { + // remove this zero skip by combining prev. and next read. + list::iterator prev_read = it; + list::iterator next_read = it; + prev_read--; + next_read++; + + data_size_elements += (*next_read /= element_size); // next read was still in units of + // bytes + *prev_read += *next_read; // combine nextRead with prevRead + offsets.erase(it); + offsets.erase(next_read); + it = prev_read; // point to prevRead now + toggle_on_read = true; // and mark that toggle flag that current iteration deal with + // read. + } + } + + // now optimise access performance by converting the offsets list to a vector. + + vector voffsets(offsets.begin(), offsets.end()); + swap(voffsets); + + // compute the finalSkip, which brings the filePointer back to the starting location, but of + // the nextFrame + FileOffset final_skip = last_offset + first_offset; + + if (final_skip) + push_back(final_skip); +} + +void FrameParser::ReadOptimiser::update_offsets(size_t new_size) { + data_size_elements = new_size * data_size_elements / element_size; + for (iterator i = begin(); i != end(); i++) { // iterate through all read offsets + // first reduce the read offsets to byte numbers, then convert it to the new element + // size + *i = new_size * (*i) / element_size; + i++; + if (i == end()) + break; + } +} + +} // namespace binary_reader +} // namespace bbp diff --git a/tools/converter/binary_reader/ReadManager.h b/tools/converter/binary_reader/binary_reader.h similarity index 57% rename from tools/converter/binary_reader/ReadManager.h rename to tools/converter/binary_reader/binary_reader.h index 76b86ea..ed62399 100644 --- a/tools/converter/binary_reader/ReadManager.h +++ b/tools/converter/binary_reader/binary_reader.h @@ -1,5 +1,4 @@ -#ifndef _ReadManager__ -#define _ReadManager__ +#pragma once /** For reference: git@bbpgitlab.epfl.ch:hpc/reportinglib.git */ #include @@ -14,9 +13,8 @@ #include #include -#include "BinaryPositions.h" - -#include "crossPlatformConversion.h" +#include "binary_positions.h" +#include "cross_platform_conversion.h" /*Terminology: * Compartment: lowest subdivision of cell. (cannot be broken down further) @@ -27,7 +25,8 @@ * See documentation for further naming conventions. */ -namespace bbpReader { +namespace bbp { +namespace binary_reader { // class list: class Header; @@ -35,13 +34,10 @@ class CellInfo; class FrameInfo; class CompartmentMapping; class SectionData; -// class SectionData::Identifier; class Identifier; class CellData; class ReadManager; class FrameParser; -// class FrameParser::FrameOrganiser; -// class FrameParser::ReadOptimiser; // end class list // type conventions: @@ -74,12 +70,11 @@ enum SortingKey { SID, LOC }; // sorting specifier for sections. namespace baseTypes { // these typedefs define from which STL some of the clases are derived from -typedef std::set FrameInfo; +typedef std::set FrameInfo; typedef std::vector CompartmentMapping; typedef std::vector SectionData; -// typedef std::map // CellData; done blow -typedef std::map FrameOrganiser; +typedef std::map FrameOrganiser; typedef std::vector ReadOptimiser; } // namespace baseTypes @@ -88,81 +83,81 @@ typedef std::vector ReadOptimiser; class Header { private: - bool isNative; + bool is_native; // header info - uint32_t headerSize; - std::string libraryVersion; - std::string simulatorVersion; - CellCount amountOfCells; - FrameDataCount totalAmountOfCompartments; - FrameCount numberOfSteps; - Time startTime; - Time endTime; + uint32_t header_size; + std::string library_version; + std::string simulator_version; + CellCount amount_of_cells; + FrameDataCount total_amount_of_compartments; + FrameCount number_of_steps; + Time start_time; + Time end_time; Time dt; - std::string dataUnit; - std::string timeUnit; - FrameDataCount mappingSize; - std::string mappingName; - FrameDataCount extraMappingSize; - std::string extraMappingName; - std::string targetName; + std::string data_unit; + std::string time_unit; + FrameDataCount mapping_size; + std::string mapping_name; + FrameDataCount extra_mapping_size; + std::string extra_mapping_name; + std::string target_name; public: Header() - : headerSize(0) {} - Header(char* readBuffer); + : header_size(0) {} + Header(char* read_buffer); // Header(const Header& source); bool operator==(const Header& h1) const; // get methods: - inline bool sourceIsNative() const { - return isNative; + inline bool source_is_native() const { + return is_native; } - inline std::string getLibraryVersion() const { - return libraryVersion; + inline std::string get_library_version() const { + return library_version; } - inline std::string getSimulatorVersion() const { - return simulatorVersion; + inline std::string get_simulator_version() const { + return simulator_version; } - inline CellCount getNumberOfCells() const { - return amountOfCells; + inline CellCount get_number_of_cells() const { + return amount_of_cells; } - inline FrameDataCount getTotalNumberOfCompartments() const { - return totalAmountOfCompartments; + inline FrameDataCount get_total_number_of_compartments() const { + return total_amount_of_compartments; } - inline FrameCount getNumberOfSteps() const { - return numberOfSteps; + inline FrameCount get_number_of_steps() const { + return number_of_steps; } - inline std::string getMappingName() const { - return mappingName; + inline std::string get_mapping_name() const { + return mapping_name; } - inline std::string getExtraMappingName() const { - return extraMappingName; + inline std::string get_extra_mapping_name() const { + return extra_mapping_name; } - inline std::string getDataUnit() const { - return dataUnit; + inline std::string get_data_unit() const { + return data_unit; } - inline std::string getTimeUnit() const { - return timeUnit; + inline std::string get_time_unit() const { + return time_unit; } - inline FrameDataCount getMappingSize() const { - return mappingSize; + inline FrameDataCount get_mapping_size() const { + return mapping_size; } - inline FrameDataCount getExtraMappingSize() const { - return extraMappingSize; + inline FrameDataCount get_extra_mapping_size() const { + return extra_mapping_size; } - inline std::string getTargetName() const { - return targetName; + inline std::string get_target_name() const { + return target_name; } - inline Time getStartTime() const { - return startTime; + inline Time get_start_time() const { + return start_time; } - inline Time getEndTime() const { - return endTime; + inline Time get_end_time() const { + return end_time; } - inline Time getTimeStepSize() const { + inline Time get_time_step_size() const { return dt; } // end get methods @@ -171,12 +166,12 @@ class Header class CellInfo { private: - CellID cellNum; - CellDataCount amountOfCompartments; + CellID cell_num; + CellDataCount amount_of_compartments; - std::ios::off_type dataLocation; - std::ios::off_type extraMappingLocation; - std::ios::off_type mappingLocation; + std::ios::off_type data_location; + std::ios::off_type extra_mapping_location; + std::ios::off_type mapping_location; void* master; @@ -184,7 +179,7 @@ class CellInfo enum key { GID, LOCATION }; CellInfo(); - CellInfo(char* readBuffer, bool isNative = true, key cmp = GID); + CellInfo(char* read_buffer, bool is_native = true, key cmp = GID); bool operator==(const CellInfo& c1) const; bool operator!=(const CellInfo& c1) const; @@ -193,27 +188,27 @@ class CellInfo bool operator<(const CellInfo& c1) const; bool operator>(const CellInfo& c1) const; - inline std::ios::off_type getDataLocation() const { - return dataLocation; + inline std::ios::off_type get_data_location() const { + return data_location; } - inline std::ios::off_type getMappingLocation() const { - return mappingLocation; + inline std::ios::off_type get_mapping_location() const { + return mapping_location; } - inline std::ios::off_type getExtraMappingLocation() const { - return extraMappingLocation; + inline std::ios::off_type get_extra_mapping_location() const { + return extra_mapping_location; } - inline CellID getCellNum() const { - return cellNum; + inline CellID get_cell_num() const { + return cell_num; } - inline CellDataCount getAmountOfCompartments() const { - return amountOfCompartments; + inline CellDataCount get_amount_of_compartments() const { + return amount_of_compartments; } - static bool compareByOffset(const CellInfo& c1, const CellInfo& c2); + static bool compare_by_offset(const CellInfo& c1, const CellInfo& c2); }; class FrameInfo: private baseTypes::FrameInfo @@ -221,17 +216,17 @@ class FrameInfo: private baseTypes::FrameInfo private: typedef baseTypes::FrameInfo Base; typedef std::map Lookup; - Lookup gidLookup; + Lookup gid_lookup; public: typedef Base::const_iterator const_iterator; FrameInfo() {} FrameInfo(char* buffer, - char* endOfBuffer, - bool isNative = true, - CellInfo::key sortMode = CellInfo::GID); - FrameInfo& filterGIDs(const std::vector& gids); + char* end_of_buffer, + bool is_native = true, + CellInfo::key sort_mode = CellInfo::GID); + FrameInfo& filter_gids(const std::vector& gids); /** * Copy constructor. Needed to ensure set iterators reference the correct collection @@ -244,7 +239,7 @@ class FrameInfo: private baseTypes::FrameInfo FrameInfo& operator=(const FrameInfo& rhs); inline const CellInfo& operator[](CellID gid) { - return *gidLookup[gid]; + return *gid_lookup[gid]; } inline const_iterator begin() const { @@ -266,7 +261,7 @@ class CompartmentMapping: public baseTypes::CompartmentMapping CompartmentMapping(); CompartmentMapping(MappingItem* start, MappingItem* end); - CompartmentMapping(MappingItem* buffer, SizeType bufSize); + CompartmentMapping(MappingItem* buffer, SizeType buf_size); }; class SectionData: public baseTypes::SectionData @@ -277,17 +272,17 @@ class SectionData: public baseTypes::SectionData public: class Identifier; SectionData(); - SectionData(DataItem firstData); + SectionData(DataItem first_data); SectionData(DataItem* start, DataItem* end); bool operator==(const SectionData& s1); bool operator!=(const SectionData& s1); - inline SectionDataCount numOfCompartments() { + inline SectionDataCount num_of_compartments() { return size(); } - static bool compareByOriginalLocation(const Identifier& c1, const Identifier& c2); + static bool compare_by_original_location(const Identifier& c1, const Identifier& c2); }; class SectionData::Identifier @@ -336,17 +331,18 @@ class SectionData::Identifier return sid; } - inline MappingItem getSID() const { + inline MappingItem get_sid() const { return sid; } - inline SectionIndex getLocation() const { + inline SectionIndex get_location() const { return loc; } }; namespace baseTypes { -typedef std::map CellData; +typedef std::map + CellData; } class CellData: public baseTypes::CellData @@ -355,22 +351,22 @@ class CellData: public baseTypes::CellData typedef baseTypes::CellData Base; typedef std::map Lookup; - SortingKey sectSort; - Lookup sidLookup; + SortingKey sect_sort; + Lookup sid_lookup; public: CellData(); - CellData(const SortingKey segmentSorting = SID); - CellData(const DataItem* dataBuffer, const CompartmentMapping& mapping); - void fillData(const DataItem* dataBuffer, const CompartmentMapping& mapping); + CellData(const SortingKey segment_sorting = SID); + CellData(const DataItem* data_buffer, const CompartmentMapping& mapping); + void fill_data(const DataItem* data_buffer, const CompartmentMapping& mapping); inline iterator find(const SectionData::Identifier& i) { return Base::find(i); } inline iterator find(const MappingItem& map_find) { - Lookup::iterator match = sidLookup.find(map_find); - if (match == sidLookup.end()) + Lookup::iterator match = sid_lookup.find(map_find); + if (match == sid_lookup.end()) return end(); else return match->second; @@ -389,31 +385,31 @@ class ReadManager: public std::ifstream typedef Base::pos_type FilePosition; private: - FileOffset cellInfoOrigin; // position in file where cell info starts (after header) + FileOffset cell_info_origin; // position in file where cell info starts (after header) - bool isNative; // indicates whether file is originated from an architecture that arranges - // variables in same byte order + bool is_native; // indicates whether file is originated from an architecture that arranges + // variables in same byte order - FrameDataCount mappingSize_floats; + FrameDataCount mapping_size_floats; - FileOffset startLocation; - FileOffset mappingLocation; + FileOffset start_location; + FileOffset mapping_location; - inline void seekgStart() { - seekg(startLocation, beg); + inline void seekg_start() { + seekg(start_location, beg); } - static CellInfo makeNullInfo(); + static CellInfo make_null_info(); protected: - inline FileOffset getStartOffset() { - return startLocation; + inline FileOffset get_start_offset() { + return start_location; } public: static const CellInfo _null_info; - Header fileInfo; + Header file_info; ReadManager(); @@ -422,54 +418,54 @@ class ReadManager: public std::ifstream * * @param filePath The bbp binary report file to open */ - explicit ReadManager(const std::string& filePath); + explicit ReadManager(const std::string& file_path); /** * Opens the filename specified provided this object has not already opened a file * - * @param filePath The bbp binary report file to open + * @param file_path The bbp binary report file to open */ - void open(const std::string& filePath); + void open(const std::string& file_path); // Find the cell's info which corresponds to cellNum (note: cellNum is not the cell index in // the file) - CellInfo retrieveFindCell(CellID cellNum); + CellInfo retrieve_find_cell(CellID cell_num); // Retrieve global (time invariant) information of a cell (not actual data of the cell) - CellInfo retrieveCellInfo(CellIndex cellIndex, CellInfo::key sortMode = CellInfo::LOCATION); + CellInfo retrieve_cell_info(CellIndex cell_index, CellInfo::key sort_mode = CellInfo::LOCATION); // NOTE: its highly recommended to either disable cache (set to 0) or set cache size to // unlimited (set to -1) before using this method: - FrameInfo retrieveAllCellInfo(CellInfo::key sortMode = CellInfo::LOCATION); + FrameInfo retrieve_all_cell_info(CellInfo::key sort_mode = CellInfo::LOCATION); /*seekg*: * all the seekg functions below are wrapping around the original iostream seekg method. */ - inline void seekgCellInfo(CellIndex cellIndex = 0) { - seekg(cellInfoOrigin); - if (cellIndex) - seekg(cellIndex * SIZE_CELL_INFO_LENGTH, cur); + inline void seekg_cell_info(CellIndex cell_index = 0) { + seekg(cell_info_origin); + if (cell_index) + seekg(cell_index * SONATA_REPORT_SIZE_CELL_INFO_LENGTH, cur); } - inline void seekgData(const CellInfo& cellSpec, FrameIndex timeStep = 0) { - seekg(cellSpec.getDataLocation() + timeStep * getFrameSize_bytes(), beg); + inline void seekg_data(const CellInfo& cell_spec, FrameIndex time_step = 0) { + seekg(cell_spec.get_data_location() + time_step * get_frame_size_bytes(), beg); } - inline void seekgMapping(const CellInfo& cellSpec) { - seekg(cellSpec.getMappingLocation(), beg); + inline void seekg_mapping(const CellInfo& cell_spec) { + seekg(cell_spec.get_mapping_location(), beg); } - inline void seekgExtraMapping(const CellInfo& cellSpec) { - seekg(cellSpec.getExtraMappingLocation(), beg); + inline void seekg_extra_mapping(const CellInfo& cell_spec) { + seekg(cell_spec.get_extra_mapping_location(), beg); } - inline void seekgMappingStart() { - seekg(mappingLocation, beg); + inline void seekg_mapping_start() { + seekg(mapping_location, beg); } - inline void seekgFrame(FrameIndex timeStep) { - seekg(startLocation + getFrameSize_bytes() * timeStep); + inline void seekg_frame(FrameIndex time_step) { + seekg(start_location + get_frame_size_bytes() * time_step); } /*read(): @@ -491,12 +487,12 @@ class ReadManager: public std::ifstream ReadManager& read(datatype* buffer, std::streamsize num) { Base::read((char*) buffer, num * sizeof(datatype)); - if (sizeof(datatype) != 1 && !isNative) // a good compiler should not include this - // entire if statement when this function is - // reading single byte items. + if (sizeof(datatype) != 1 && !is_native) // a good compiler should not include this + // entire if statement when this function is + // reading single byte items. // This if is to avoid entering an empty loop. Otherwise could have been done in // switch below. - for (datatype* endOfBuffer = buffer + num; buffer < endOfBuffer; buffer++) + for (datatype* end_of_buffer = buffer + num; buffer < end_of_buffer; buffer++) switch (sizeof(datatype)) { // metamorphic switch case 8: // double, int64_t, offsets etc DOUBLESWAP(*buffer); @@ -520,7 +516,7 @@ class ReadManager: public std::ifstream * and exit execution if an error occured. */ - void checkState(); + void check_state(); /*getFrameSize: * returns the number of bytes that hold the data of all compartments of all cells for a @@ -529,8 +525,8 @@ class ReadManager: public std::ifstream * use fileInfo.getTotalNumberOfCompartments() instead for number of floats. */ - inline std::streamsize getFrameSize_bytes() const { - return fileInfo.getTotalNumberOfCompartments() * sizeof(DataItem); + inline std::streamsize get_frame_size_bytes() const { + return file_info.get_total_number_of_compartments() * sizeof(DataItem); } /*getMappingSize: @@ -539,8 +535,8 @@ class ReadManager: public std::ifstream * please note that this is in bytes not floats. */ - inline std::streamsize getMappingSize_bytes() const { - return mappingSize_floats * sizeof(MappingItem); + inline std::streamsize get_mapping_size_bytes() const { + return mapping_size_floats * sizeof(MappingItem); } // The following are low level c-style reading interfaces for max performance - they provide @@ -553,20 +549,20 @@ class ReadManager: public std::ifstream * (NOT float). */ - void readFrameMapping(MappingItem* buffer); + void read_frame_mapping(MappingItem* buffer); /*readFrame: - * will fill dataBuffer with data from next frame. - * use seekgFrame() to specify timestep. - * use fileInfo.getTotalNumberOfCompartments() to determine the size of buffer + * will fill data_buffer with data from next frame. + * use seekg_frame() to specify timestep. + * use fileInfo.get_total_number_of_compartments() to determine the size of buffer */ - void readFrame(DataItem* dataBuffer); + void read_frame(DataItem* data_buffer); private: /** * Upon opening a report file, read the header with metadata */ - void readHeader(); + void read_header(); }; class FrameParser: protected ReadManager @@ -579,89 +575,89 @@ class FrameParser: protected ReadManager class FrameOrganiser: public baseTypes::FrameOrganiser { private: - SortingKey sectSort; + SortingKey sect_sort; public: FrameOrganiser(); - explicit FrameOrganiser(SortingKey sectionSorting); + explicit FrameOrganiser(SortingKey section_sorting); - void fillData(const FrameInfo& info, - const std::vector& mapping, - const float* dataBuffer); + void fill_data(const FrameInfo& info, + const std::vector& mapping, + const float* data_buffer); - void makeOrderedDataBuffer(DataItem* buffer) const; + void make_ordered_data_buffer(DataItem* buffer) const; - CompartmentIndexing makeOrderedOffsetReferences() const; - CompartmentCounts makeOrderedSegSizes() const; + CompartmentIndexing make_ordered_offset_references() const; + CompartmentCounts make_ordered_seg_sizes() const; }; class ReadOptimiser: public baseTypes::ReadOptimiser { private: - void updateOffsets(size_t newSize); + void update_offsets(size_t new_size); protected: - FileOffset firstOffset; - FrameDataCount dataSize_elements; + FileOffset first_offset; + FrameDataCount data_size_elements; - size_t elementSize; - static size_t defaultElementSize; + size_t element_size; + static size_t default_element_size; public: ReadOptimiser(); - explicit ReadOptimiser(std::streamsize frameSize_bytes); - ReadOptimiser(const FrameInfo& cellsToRead, - std::streamsize frameSize_bytes, - FilePosition startLocation); + explicit ReadOptimiser(std::streamsize frame_size_bytes); + ReadOptimiser(const FrameInfo& cells_to_read, + std::streamsize frame_size_bytes, + FilePosition start_location); - inline FileOffset getFirst() const { - return firstOffset; + inline FileOffset get_first() const { + return first_offset; } - inline FrameDataCount getDataSize_elements() const { - return dataSize_elements; + inline FrameDataCount get_data_size_elements() const { + return data_size_elements; } - inline void setElementSize(size_t newSize) { - if (newSize != elementSize) { - updateOffsets(newSize); - elementSize = newSize; + inline void set_element_size(size_t new_size) { + if (new_size != element_size) { + update_offsets(new_size); + element_size = new_size; } } - static inline void setDefaultElementSize(size_t newDefault) { - defaultElementSize = newDefault; + static inline void set_default_element_size(size_t new_default) { + default_element_size = new_default; } - inline size_t getElementSize() const { - return elementSize; + inline size_t get_element_size() const { + return element_size; } }; - void readFrame(DataItem* buffer); + void read_frame(DataItem* buffer); private: - DataItem* readBuffer; - DataItem** refArray; + DataItem* read_buffer; + DataItem** ref_array; ReadOptimiser offsets; FrameOrganiser org; FrameIndex timestep; - void createReferences(const FrameInfo& cinfo, bool sortData); + void create_references(const FrameInfo& cinfo, bool sort_data); - void bufferTransfer(DataItem* target) const; + void buffer_transfer(DataItem* target) const; public: /*Constructor: * Will read next frame of data and parse it into a map of CellData structure from file. */ FrameParser(); - FrameParser(const std::string& file, bool sortData = true); + FrameParser(const std::string& file, bool sort_data = true); FrameParser(const std::string& file, - const std::vector& gidTarget, - bool sortData = true); + const std::vector& gid_target, + bool sort_data = true); ~FrameParser(); /*retarget: @@ -671,17 +667,17 @@ class FrameParser: protected ReadManager */ inline FrameIndex simtime2index(Time time) { - return (FrameIndex)((time - fileInfo.getStartTime()) / fileInfo.getTimeStepSize()); + return (FrameIndex)((time - file_info.get_start_time()) / file_info.get_time_step_size()); } - void retarget(const std::vector& gidTarget, bool sortData = true); + void retarget(const std::vector& gid_target, bool sort_data = true); - inline const Header& getHeader() const { - return fileInfo; + inline const Header& get_header() const { + return file_info; } - inline FrameDataCount getBufferSize_elements() const { - return offsets.getDataSize_elements(); + inline FrameDataCount get_buffer_size_elements() const { + return offsets.get_data_size_elements(); } // to select frame (similar use to iterators): @@ -692,33 +688,33 @@ class FrameParser: protected ReadManager FrameParser& operator--(int); // decrement timestep FrameParser& operator+=(FrameDiff increment); // increment timestep by scalar FrameParser& operator-=(FrameDiff decrement); // decrement timestep by scalar - FrameParser& operator=(FrameIndex newTime); // set timestep to scalar + FrameParser& operator=(FrameIndex new_time); // set timestep to scalar // returns the current timestep: - inline FrameIndex getTimestep() const { + inline FrameIndex get_timestep() const { return timestep; } - inline bool hasMore() const { - return timestep < fileInfo.getNumberOfSteps(); + inline bool has_more() const { + return timestep < file_info.get_number_of_steps(); } // returns the current time that the current timestep represents. - Time getTime() const; + Time get_time() const; - void readFrameMapping(MappingItem* buffer); - void readFrameData(DataItem* buffer); // reads data of next frame and advances to the frame - // after. Data is sorted by GID and segment num + void read_frame_mapping(MappingItem* buffer); + void read_frame_data(DataItem* buffer); // reads data of next frame and advances to the frame + // after. Data is sorted by GID and segment num - /*getReferences() + /*get_references() * returns a vector of vectors of offsets that correspond to the relevant segments inside * the read buffer. * eg. - * getReferences()[12][31]; //will give the offset for segment 31 in cell 12 + * get_references()[12][31]; //will give the offset for segment 31 in cell 12 * if the segment has no comparments, it will return an offset of 0. */ - inline CompartmentIndexing getReferences() const { - return org.makeOrderedOffsetReferences(); + inline CompartmentIndexing get_references() const { + return org.make_ordered_offset_references(); } /*getCompartmentCounts() @@ -726,9 +722,9 @@ class FrameParser: protected ReadManager * but instead of containing the offsets for each segment, * it contains the amount of compartmetns of the corresponding segment. */ - inline CompartmentCounts getCompartmentCounts() const { - return org.makeOrderedSegSizes(); + inline CompartmentCounts get_compartment_counts() const { + return org.make_ordered_seg_sizes(); } }; -} // namespace bbpReader -#endif +} // namespace binary_reader +} // namespace bbp diff --git a/tools/converter/binary_reader/crossPlatformConversion.h b/tools/converter/binary_reader/cross_platform_conversion.h similarity index 87% rename from tools/converter/binary_reader/crossPlatformConversion.h rename to tools/converter/binary_reader/cross_platform_conversion.h index b14b9ee..7724acd 100644 --- a/tools/converter/binary_reader/crossPlatformConversion.h +++ b/tools/converter/binary_reader/cross_platform_conversion.h @@ -1,7 +1,6 @@ // Author: Dr Konstantinos Sfyrakis -#ifndef CROSSPLATFORMCONVERSION_H_ -#define CROSSPLATFORMCONVERSION_H_ +#pragma once #define SMALL_NUM 0.00000001 // anything that avoids division overflow #define ABS(x) ((x) >= 0 ? (x) : -(x)) // absolute value @@ -24,5 +23,3 @@ inline void ByteSwap(unsigned char* b, int n) { i++, j--; } } - -#endif /*CROSSPLATFORMCONVERSION_H_*/ diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index ff95854..94696e2 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -8,11 +8,11 @@ #include #endif -#include "binary_reader/ReadManager.h" +#include "binary_reader/binary_reader.h" #include #include -using namespace bbpReader; +using namespace bbp::binary_reader; static void show_usage(std::string name) { std::cerr << "Usage: " << name << " [population_name]\n" @@ -62,33 +62,33 @@ int main(int argc, char* argv[]) { } FrameParser frame_parser(file_name); - Header header = frame_parser.getHeader(); + Header header = frame_parser.get_header(); std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); // Get header information in order to create the report - double tstart = header.getStartTime(); - double tstop = header.getEndTime(); - double dt = header.getTimeStepSize(); + double tstart = header.get_start_time(); + double tstop = header.get_end_time(); + double dt = header.get_time_step_size(); logger->info("Report info: tstart = '{}', tstop = '{}', dt = '{}'", tstart, tstop, dt); sonata_create_report(report_name.data(), tstart, tstop, dt, "mV", "compartment"); // Get Cell information to create node/element structure // TODO: check if it could be done without getting the FrameParser per gid ReadManager file(file_name); - FrameInfo cells = file.retrieveAllCellInfo(); + FrameInfo cells = file.retrieve_all_cell_info(); std::vector node_ids(1); for (FrameInfo::const_iterator it = cells.begin(); it != cells.end(); ++it) { - node_ids[0] = it->getCellNum(); - sonata_add_node(report_name.data(), population_name.data(), 0, it->getCellNum()); + node_ids[0] = it->get_cell_num(); + sonata_add_node(report_name.data(), population_name.data(), 0, it->get_cell_num()); if (report_type == "--soma") { sonata_add_element(report_name.data(), population_name.data(), node_ids[0], 0, nullptr); } else { // --compartment FrameParser frame_parser_gid(file_name, node_ids); - int num_element_ids = frame_parser_gid.getBufferSize_elements(); + int num_element_ids = frame_parser_gid.get_buffer_size_elements(); DataItem* element_ids_buffer = new DataItem[num_element_ids]; - frame_parser_gid.readFrameMapping(element_ids_buffer); + frame_parser_gid.read_frame_mapping(element_ids_buffer); std::vector element_ids(element_ids_buffer, element_ids_buffer + num_element_ids); @@ -101,12 +101,12 @@ int main(int argc, char* argv[]) { // Generate the initial structure sonata_prepare_datasets(); - uint64_t element_ids_per_frame = frame_parser.getBufferSize_elements(); + uint64_t element_ids_per_frame = frame_parser.get_buffer_size_elements(); DataItem* element_ids_buffer = new DataItem[element_ids_per_frame]; uint32_t timestep = 0; // Write the timestep frames - while (frame_parser.hasMore()) { - frame_parser.readFrameData(element_ids_buffer); + while (frame_parser.has_more()) { + frame_parser.read_frame_data(element_ids_buffer); sonata_write_buffered_data(report_name.data(), element_ids_buffer, element_ids_per_frame, From 189200562225c58b5929586bd0903dd68416a03c Mon Sep 17 00:00:00 2001 From: Jorge Blanco Alonso Date: Wed, 12 Oct 2022 12:38:34 +0200 Subject: [PATCH 10/10] Use std::vector instead of raw pointers --- tools/converter/reports_converter.cpp | 16 +++++++--------- tools/converter/spikes_converter.cpp | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tools/converter/reports_converter.cpp b/tools/converter/reports_converter.cpp index 94696e2..3516b88 100644 --- a/tools/converter/reports_converter.cpp +++ b/tools/converter/reports_converter.cpp @@ -14,7 +14,7 @@ using namespace bbp::binary_reader; -static void show_usage(std::string name) { +static void show_usage(const std::string& name) { std::cerr << "Usage: " << name << " [population_name]\n" << "Options:\n" << "\t-h,--help\t\tShow this help message\n" @@ -64,7 +64,7 @@ int main(int argc, char* argv[]) { FrameParser frame_parser(file_name); Header header = frame_parser.get_header(); - std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); + const std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); // Get header information in order to create the report double tstart = header.get_start_time(); @@ -87,11 +87,9 @@ int main(int argc, char* argv[]) { } else { // --compartment FrameParser frame_parser_gid(file_name, node_ids); int num_element_ids = frame_parser_gid.get_buffer_size_elements(); - DataItem* element_ids_buffer = new DataItem[num_element_ids]; - frame_parser_gid.read_frame_mapping(element_ids_buffer); + std::vector element_ids(num_element_ids); + frame_parser_gid.read_frame_mapping(element_ids.data()); - std::vector element_ids(element_ids_buffer, - element_ids_buffer + num_element_ids); for (auto& element : element_ids) { sonata_add_element( report_name.data(), population_name.data(), node_ids[0], element, nullptr); @@ -102,13 +100,13 @@ int main(int argc, char* argv[]) { sonata_prepare_datasets(); uint64_t element_ids_per_frame = frame_parser.get_buffer_size_elements(); - DataItem* element_ids_buffer = new DataItem[element_ids_per_frame]; + std::vector element_ids_buffer(element_ids_per_frame); uint32_t timestep = 0; // Write the timestep frames while (frame_parser.has_more()) { - frame_parser.read_frame_data(element_ids_buffer); + frame_parser.read_frame_data(element_ids_buffer.data()); sonata_write_buffered_data(report_name.data(), - element_ids_buffer, + element_ids_buffer.data(), element_ids_per_frame, 1); // Number of frames to write if (timestep % 1000 == 0 && timestep > 0) { diff --git a/tools/converter/spikes_converter.cpp b/tools/converter/spikes_converter.cpp index 4a1e147..223d1d1 100644 --- a/tools/converter/spikes_converter.cpp +++ b/tools/converter/spikes_converter.cpp @@ -10,7 +10,7 @@ #include #include -static void show_usage(std::string name) { +static void show_usage(const std::string& name) { std::cerr << "Usage: " << name << " [population_name]\n" << "Options:\n" << "\t-h,--help\t\tShow this help message\n" @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) { } // Create a spike file - std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); + const std::string report_name = file_name.substr(file_name.find_last_of("/\\") + 1); sonata_create_spikefile(".", report_name.data()); uint64_t population_offset = 0;