Initial implementation of converter bin to SONATA reports

.github/workflows/run_test.yaml

@@ -10,8 +10,8 @@ jobs:
             matrix:
                 os: [ ubuntu-18.04 ]
                 config:
-                  - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON", mpi: ON}
-                  - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=OFF", mpi: OFF}
+                  - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_CONVERTER=ON", mpi: ON}
+                  - {cmake_option: "-DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=OFF -DSONATA_REPORT_ENABLE_CONVERTER=ON", mpi: OFF}
         steps:
             - name: Checkout repository
               uses: actions/checkout@v2

CMakeLists.txt

@@ -43,6 +43,8 @@ option(SONATA_REPORT_ENABLE_SUBMODULES "Use Git submodules for header-only depen
 option(SONATA_REPORT_ENABLE_WARNING_AS_ERROR "Compile C++ with warnings as errors" OFF)
 option(SONATA_REPORT_ENABLE_MPI "Enable MPI-based execution" ON)
 option(SONATA_REPORT_ENABLE_TEST "Enable tests" ON)
+option(SONATA_REPORT_ENABLE_CONVERTER "Enable binary report converter" OFF)
+
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/CMake)
 set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
@@ -95,7 +97,7 @@ if(SONATA_REPORT_ENABLE_MPI)
     else()
         if (HDF5_FOUND)
             # Avoid NOTFOUND error
-            set(MPI_CXX_LIBRARIES "")
+            set(MPI_C_LIBRARIES "")
             message(STATUS "HDF5 found, Using reporting serial implementation")
         else()
             message(FATAL_ERROR "No MPI or HDF5 found")
@@ -157,3 +159,10 @@ if(SONATA_REPORT_ENABLE_TEST)
         )
     endif()
 endif()
+
+# =============================================================================
+# Converter of binary to SONATA reports
+# =============================================================================
+if(SONATA_REPORT_ENABLE_CONVERTER)
+    add_subdirectory(tools/converter)
+endif()

include/bbp/sonata/reports.h

@@ -119,6 +119,15 @@ int sonata_record_node_data(double step,
  */
 int sonata_record_data(double step);
 
+/**
+ * \brief Write steps_to_write steps given a pre-buffered data
+ * \return -3 if the Sonata report doesn't exist, -1 if the report name doesn't exist, 0 otherwise
+ */
+int sonata_write_buffered_data(const char* report_name,
+                               const float* buffered_data,
+                               size_t buffered_data_size,
+                               uint32_t steps_to_write);
+
 /**
  * \brief Check status of the recordings/buffers and flush if necessary
  * \return 0

src/CMakeLists.txt

@@ -40,7 +40,7 @@ set_target_properties(sonata_report
 )
 target_link_libraries(sonata_report
     PRIVATE spdlog::spdlog_header_only
-    PRIVATE ${MPI_CXX_LIBRARIES}
+    PRIVATE ${MPI_C_LIBRARIES}
     PRIVATE ${HDF5_C_LIBRARIES}
 )
 

src/data/sonata_data.cpp

@@ -163,7 +163,7 @@ void SonataData::record_data(double step) {
     last_step_recorded_ += reporting_period_;
 
     if (current_step_ == steps_to_write_) {
-        write_data();
+        flush();
     }
 }
 
@@ -199,7 +199,7 @@ void SonataData::check_and_write(double timestep) {
                 remaining_steps_,
                 steps_recorded_);
         }
-        write_data();
+        flush();
     }
     steps_recorded_ = 0;
 }
@@ -313,26 +313,30 @@ void SonataData::write_spike_populations() {
     }
 }
 
-void SonataData::write_data() {
+void SonataData::write_data(const std::vector<float>& buffered_data, uint32_t steps_to_write) {
     if (remaining_steps_ <= 0) {  // Nothing left to write
         return;
     }
-    if (current_step_ >= remaining_steps_) {  // Avoid writing out of bounds
-        current_step_ = remaining_steps_;
+    if (steps_to_write  >= remaining_steps_) {  // Avoid writing out of bounds
+        steps_to_write  = remaining_steps_;
     }
     if (SonataReport::rank_ == 0) {
         logger->debug("WRITING timestep data to file {} in population {}",
                       report_name_,
                       population_name_);
     }
-    hdf5_writer_->write_2D(report_buffer_, current_step_, total_elements_);
-    remaining_steps_ -= current_step_;
+    hdf5_writer_->write_2D(buffered_data, steps_to_write , total_elements_);
+    remaining_steps_ -= steps_to_write ;
     if (SonataReport::rank_ == 0) {
-        logger->debug("\t-Steps written: {}", current_step_);
+        logger->debug("\t-Steps written: {}", steps_to_write );
         logger->debug("\t-Remaining steps: {}", remaining_steps_);
     }
     last_position_ = 0;
-    current_step_ = 0;
+    steps_to_write  = 0;
+}
+
+void SonataData::flush() {
+    write_data(report_buffer_, current_step_);
 }
 
 void SonataData::close() {

src/data/sonata_data.h

@@ -73,7 +73,8 @@ class SonataData
     void write_spike_populations();
     void add_population(std::unique_ptr<Population>&& population);
 
-    void write_data();
+    void write_data(const std::vector<float>& buffered_data, uint32_t steps_to_write);
+    void flush();
     void close();
 
     bool is_due_to_report(double step) const noexcept;

src/library/report.cpp

@@ -113,6 +113,12 @@ void Report::record_data(double step) {
     }
 }
 
+void Report::write_buffered_data(const std::vector<float>& buffered_data, uint32_t steps_to_write) {
+    for (const auto& sonata_data : sonata_populations_) {
+        sonata_data->write_data(buffered_data, steps_to_write);
+    }
+}
+
 void Report::check_and_flush(double timestep) {
     for (const auto& sonata_data : sonata_populations_) {
         sonata_data->check_and_write(timestep);
@@ -134,7 +140,7 @@ void Report::flush(double time) {
     }
     for (const auto& sonata_data : sonata_populations_) {
         // Write if there are any remaining steps to write
-        sonata_data->write_data();
+        sonata_data->flush();
         if (time - tend_ + dt_ / 2 > 1e-6) {
             sonata_data->close();
         }

src/library/report.h

@@ -45,6 +45,8 @@ class Report
 
     virtual void record_data(double step, const std::vector<uint64_t>& node_ids);
     virtual void record_data(double step);
+    virtual void write_buffered_data(const std::vector<float>& buffered_data,
+                                     uint32_t steps_to_write);
     virtual void check_and_flush(double timestep);
     virtual void flush(double time);
     void refresh_pointers(std::function<double*(double*)> refresh_function);

src/library/reports.cpp

@@ -104,6 +104,22 @@ int sonata_record_data(double step) {
     return 0;
 }
 
+int sonata_write_buffered_data(const char* report_name,
+                               const float* buffered_data,
+                               size_t buffered_data_size,
+                               uint32_t steps_to_write) {
+    if (sonata_report.is_empty()) {
+        return -3;
+    }
+    if (!sonata_report.report_exists(report_name)) {
+        return -1;
+    }
+    const std::vector<float> data(buffered_data, buffered_data + buffered_data_size);
+    auto report = sonata_report.get_report(report_name);
+    report->write_buffered_data(data, steps_to_write);
+    return 0;
+}
+
 int sonata_check_and_flush(double timestep) {
     auto functor = std::mem_fn(&bbp::sonata::Report::check_and_flush);
     sonata_report.apply_all(functor, timestep);

tests/CMakeLists.txt

@@ -1,2 +1,5 @@
 add_subdirectory(unit)
 add_subdirectory(integration)
+if(SONATA_REPORT_ENABLE_CONVERTER)
+    add_subdirectory(converter)
+endif()

tests/converter/CMakeLists.txt

@@ -0,0 +1,6 @@
+configure_file(converter_test.sh.in converter_test.sh @ONLY)
+
+add_test(NAME converter_test
+    COMMAND "/bin/sh" ${CMAKE_CURRENT_BINARY_DIR}/converter_test.sh
+    WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+)

tests/converter/converter_test.sh.in

@@ -0,0 +1,26 @@
+#! /bin/sh
+
+@PROJECT_BINARY_DIR@/tools/converter/spikes_converter @CMAKE_CURRENT_SOURCE_DIR@/spikes.ref
+@PROJECT_BINARY_DIR@/tools/converter/reports_converter @CMAKE_CURRENT_SOURCE_DIR@/soma.ref --soma
+ref_spikes=@CMAKE_CURRENT_SOURCE_DIR@/spikes.ref.h5
+ref_soma=@CMAKE_CURRENT_SOURCE_DIR@/soma.ref.h5
+
+h5diff -c spikes.ref.h5 $ref_spikes /spikes/All/timestamps > diff_spikes.dat 2>&1
+h5diff -c spikes.ref.h5 $ref_spikes /spikes/All/node_ids > diff_spikes.dat 2>&1
+h5diff -c soma.ref.h5 $ref_soma /report/All/data > diff_soma.dat 2>&1
+
+if [ -s diff_spikes.dat ]
+then
+  echo "Spike results are different, check the file diff_spikes.dat. Test failed!"
+  cat diff_spikes.dat
+  exit 1
+elif [ -s diff_soma.dat ]
+then
+  echo "Soma results are different, check the file diff_soma.dat. Test failed!"
+  cat diff_soma.dat
+  exit 1
+else
+  echo "Results are the same, test passed"
+  rm -f *.dat
+  exit 0
+fi