Line data    Source code

       1             : #include "Common.h"
       2             : #include "Layer.h"
       3             : #include "NeuralNetworkImportExportCSV.h"
       4             : #include "NeuronConnection.h"
       5             : #include "exception/EmptyCellException.h"
       6             : #include "exception/ImportExportException.h"
       7             : #include <algorithm> // for std::transform
       8             : #include <cctype>    // for std::tolower
       9             : #include <cstddef>
      10             : #include <exception>
      11             : #include <fstream>
      12             : #include <opencv2/opencv.hpp>
      13             : #include <optional>
      14             : #include <string>
      15             : 
      16             : using namespace sipai;
      17             : 
      18           4 : void NeuralNetworkImportExportCSV::exportNeuronsWeights(
      19             :     const std::unique_ptr<NeuralNetwork> &network, const AppParams &appParams,
      20             :     std::function<void(int)> progressCallback, int progressInitialValue) const {
      21             :   // get the csv filename
      22           4 :   std::string filename = Common::getFilenameCsv(appParams.network_to_export);
      23           4 :   std::ofstream file(filename);
      24             : 
      25             :   // Write the data
      26           4 :   size_t max_weights = network->max_weights;
      27          16 :   for (size_t layer_index = 0; layer_index < network->layers.size();
      28             :        layer_index++) {
      29          12 :     const auto &layer = network->layers.at(layer_index);
      30          12 :     if (layer->layerType == LayerType::LayerInput) {
      31             :       // no weights for Input Layer, as it will be input data weights
      32           4 :       continue;
      33             :     }
      34             : 
      35          28 :     for (size_t row = 0; row < layer->neurons.size(); row++) {
      36          80 :       for (size_t col = 0; col < layer->neurons.at(row).size(); col++) {
      37          60 :         Neuron &neuron = layer->neurons[row][col];
      38             :         // Write the neuron weights, empty 3rd neighbors column then
      39          60 :         file << layer_index << "," << neuron.weights.rows << ","
      40          60 :              << neuron.weights.cols << "," << row << "," << col << ",,"
      41          60 :              << neuron.toStringCsv(max_weights) << "\n";
      42             :         // Write the neighbors connections weights
      43          60 :         file << layer_index << "," << neuron.weights.rows << ","
      44          60 :              << neuron.weights.cols << "," << row << "," << col << ","
      45          60 :              << neuron.neighbors.size() << ","
      46          60 :              << neuron.toNeighborsStringCsv(max_weights) << "\n";
      47             :       }
      48             :     }
      49             :   }
      50           4 : }
      51             : 
      52           1 : void NeuralNetworkImportExportCSV::importNeuronsWeights(
      53             :     std::unique_ptr<NeuralNetwork> &network, const AppParams &appParams,
      54             :     std::function<void(int)> progressCallback, int progressInitialValue) const {
      55             : 
      56          30 :   auto split = [](const std::string &s, char delimiter) {
      57          30 :     std::vector<std::optional<float>> tokens;
      58          30 :     std::string token;
      59          30 :     std::istringstream tokenStream(s);
      60         909 :     while (std::getline(tokenStream, token, delimiter)) {
      61        1508 :       tokens.push_back(token.empty() ? std::nullopt
      62        1508 :                                      : std::make_optional(std::stof(token)));
      63             :     }
      64          60 :     return tokens;
      65          30 :   };
      66             : 
      67             :   // get the csv filename
      68           1 :   std::string filename = Common::getFilenameCsv(appParams.network_to_import);
      69             : 
      70             :   // get the total of lines to use for progress calculus
      71           1 :   size_t totalLines = 0;
      72           1 :   if (progressCallback) {
      73           0 :     totalLines = Common::countLines(filename);
      74             :   }
      75             : 
      76           1 :   std::ifstream file(filename);
      77           1 :   if (!file.is_open()) {
      78           0 :     throw ImportExportException("Failed to open file: " + filename);
      79             :   }
      80             : 
      81             :   // parsing the csv
      82           1 :   std::string line;
      83           1 :   int oldProgressValue = progressInitialValue;
      84          31 :   for (int current_line_number = 1; std::getline(file, line);
      85             :        ++current_line_number) {
      86          30 :     const auto &fields = split(line, ',');
      87             : 
      88          30 :     if (fields.size() < 6) {
      89           0 :       throw ImportExportException("CSV parsing error at line (" +
      90           0 :                                   std::to_string(current_line_number) +
      91           0 :                                   "): invalid column numbers");
      92             :     }
      93             : 
      94          30 :     auto layer_index = static_cast<size_t>(fields[0].value());
      95          30 :     auto weights_rows = static_cast<size_t>(fields[1].value());
      96          30 :     auto weights_cols = static_cast<size_t>(fields[2].value());
      97          30 :     auto neuron_row = static_cast<size_t>(fields[3].value());
      98          30 :     auto neuron_col = static_cast<size_t>(fields[4].value());
      99          30 :     auto neighboors_count = fields[5];
     100             : 
     101          30 :     if (!neighboors_count) {
     102             :       // add the neuron weights
     103          15 :       cv::Mat weights = cv::Mat((int)weights_rows, (int)weights_cols, CV_32FC4);
     104          15 :       size_t i_cols = 0;
     105          15 :       size_t i_rows = 0;
     106          90 :       for (size_t pos = 6; pos + 4 < fields.size();
     107          75 :            pos += 4) { // pos start at fields[6], then increment of
     108             :                        // the length of cv::Vec4f (4)
     109          75 :         auto r = fields[pos];
     110          75 :         auto g = fields[pos + 1];
     111          75 :         auto b = fields[pos + 2];
     112          75 :         auto a = fields[pos + 3];
     113          75 :         if (r && g && b && a) {
     114          69 :           weights.at<cv::Vec4f>((int)i_rows, (int)i_cols) =
     115         138 :               cv::Vec4f(*r, *g, *b, *a);
     116             :         }
     117          75 :         i_cols++;
     118          75 :         if (i_cols >= weights_cols) {
     119          21 :           i_cols = 0;
     120          21 :           i_rows++;
     121             :         }
     122             :       }
     123          15 :       network->layers.at(layer_index)
     124          15 :           ->neurons.at(neuron_row)
     125          15 :           .at(neuron_col)
     126          15 :           .weights = weights;
     127          15 :     } else {
     128             :       // add the neighboors and their weights
     129             :       // add the neuron weights
     130          15 :       std::vector<cv::Vec4f> weights;
     131          90 :       for (size_t pos = 6; pos + 4 < fields.size();
     132          75 :            pos += 4) { // pos start at fields[6], then increment of
     133             :                        // the length of cv::Vec4f (4)
     134          75 :         auto r = fields[pos];
     135          75 :         auto g = fields[pos + 1];
     136          75 :         auto b = fields[pos + 2];
     137          75 :         auto a = fields[pos + 3];
     138          75 :         if (r && g && b && a) {
     139          38 :           weights.push_back(cv::Vec4f(*r, *g, *b, *a));
     140             :         }
     141             :       }
     142          15 :       auto &connections = network->layers.at(layer_index)
     143          15 :                               ->neurons[neuron_row][neuron_col]
     144          15 :                               .neighbors;
     145          15 :       if (connections.size() != weights.size()) {
     146           0 :         throw ImportExportException("CSV parsing error at line (" +
     147           0 :                                     std::to_string(current_line_number) +
     148           0 :                                     "): invalid column numbers");
     149             :       }
     150          53 :       for (size_t i = 0; i < connections.size(); i++) {
     151          38 :         connections.at(i).weight = weights.at(i);
     152             :       }
     153          15 :     }
     154             : 
     155          30 :     if (progressCallback) {
     156           0 :       int value = progressInitialValue +
     157           0 :                   ((100 * current_line_number) / (int)totalLines);
     158           0 :       if (value != oldProgressValue) {
     159           0 :         progressCallback(value);
     160           0 :         oldProgressValue = value;
     161             :       }
     162             :     }
     163          30 :   }
     164           1 : }