Parsing row by row (version 1.1.0)

Author: michalmonday

CSV_Parser.cpp

@@ -109,6 +109,28 @@ CSV_Parser::~CSV_Parser() {
   free(is_fmt_unsigned);
 }
 
+// external function declaration for feeding characters to parser it must return a char
+extern char __attribute__((weak)) feedRowParser();
+char __attribute__((weak)) feedRowParser() { return '-'; }
+// external function declaration for checking if row parser finished parsing
+extern bool __attribute__((weak)) rowParserFinished();
+bool __attribute__((weak)) rowParserFinished() { return true; }
+// both functions above must be defined by the user, weak attribute is to avoid compilation
+// fail if the user doesn't define them
+
+bool CSV_Parser::parseRow() {
+  if (rowParserFinished()) 
+    return false;
+  rows_count = 0;
+  while (!rowParserFinished() && rows_count == 0)
+    *this << feedRowParser();
+  // thanks to the line below the csv could end without '\n' and the last value 
+  // would still be parsed if the rowParserFinished() returns true
+  if (rowParserFinished())
+    parseLeftover();
+  return rows_count > 0;
+}
+
 #ifndef CSV_PARSER_DONT_IMPORT_SD
 bool CSV_Parser::readSDfile(const char *f_name) {
   // open the file. note that only one file can be open at a time,

CSV_Parser.h

@@ -120,6 +120,11 @@ class CSV_Parser {
 #ifndef CSV_PARSER_DONT_IMPORT_SD
   bool readSDfile(const char *f_name);
 #endif
+
+  /** @brief Reads a single row provided by the user-defined functions: feedRowParser (returns char to be supplied) and rowParserFinished (returns whether all rows were parsed)
+      @return true if row was parsed, false if not (e.g. if rowParserFinished() returned true) 
+     */
+  bool parseRow();
 
   int getColumnsCount();
 

README.md

@@ -11,6 +11,7 @@
     * Headerless files
     * Custom delimiter
     * Custom quote character
+    * Parsing row at a time
 * [Troubleshooting](#troubleshooting)   
     * Checking if the file was parsed correctly
     * Platformio and SD library issue  
@@ -289,6 +290,10 @@ Quote character is 5th parameter of the constructor. It's double quote (") by de
 CSV_Parser cp(csv_str, /*format*/ "sLdcfxs", /*has_header*/ true, /*delimiter*/ ',', /*quote_char*/ "'");
 ```
 
+### Parsing row at a time
+Large files often can't be stored in the limited memory of microcontrollers. For that reason it's possible to parse the file row by row.
+See the [parsing_row_by_row.ino](./examples/parsing_row_by_row/parsing_row_by_row.ino) example for more information. When deciding to parse row by row, it is suggested to not combine it with the default way of parsing. Please note that during row by row parsing the `cp.getRowsCount()` method will return 0 or 1 instead of the total number of previously parsed rows.
+
 
 ## Troubleshooting  
 

examples/parsing_row_by_row/parsing_row_by_row.ino

@@ -0,0 +1,105 @@
+/*  parsing_row_by_row example for: https://github.com/michalmonday/CSV-Parser-for-Arduino
+
+The user must define the following 2 functions/callbacks:
+- char feedRowParser()
+- bool rowParserFinished()
+
+That is because cp.parseRow() calls these functions continuously "under the hood".
+
+*/
+#include <CSV_Parser.h>
+/*
+This code prints:
+
+Accessing values by column name:
+0. String = hello
+0. Number = 5
+
+1. String = world
+1. Number = 10
+
+
+CSV_Parser content:
+rows_count = 0, cols_count = 2
+   Header:
+      my_strings | my_numbers
+   Types:
+      char* | int32_t
+   Values:
+Memory occupied by values themselves = 22
+sizeof(CSV_Parser) = 25
+
+
+ */
+ 
+
+char * csv_str = "my_strings,my_numbers\r\n"
+                 "hello,5\r\n"
+                 "world,10\r\n";
+int csv_str_index = 0;
+
+// This function is responsible for supplying characters to be parsed.
+// csv_str may be replaced with SD card reading code or any other
+// source of CSV text, like serial port or HTTP request
+char feedRowParser() {
+  return csv_str[csv_str_index++];
+}
+
+// This function must return true when the whole CSV file was supplied 
+// to feedRowParser() function. It will make sure that cp.parseRow() 
+// returns false when the end of CSV was reached.
+bool rowParserFinished() {
+  return csv_str[csv_str_index] == 0;
+}
+
+
+/*
+Alternative way to write both of these functions:
+
+bool csv_end_was_supplied = false;
+char feedRowParser() {
+  if (csv_str_index >= strlen(csv_str)) {
+    csv_end_was_supplied = true;
+    return 0;
+  }
+  return csv_str[csv_str_index++];
+}
+
+bool rowParserFinished() {
+  return csv_end_was_supplied;
+}
+ */
+ 
+void setup() {
+  Serial.begin(115200);
+  delay(5000);
+                   
+  CSV_Parser cp(/*format*/ "sL");
+
+  Serial.println("Accessing values by column name:");
+
+  // parseRow calls feedRowParser() continuously until it reads a 
+  // full row or until the rowParserFinished() returns true
+  int row_index = 0;
+  while (cp.parseRow()) {
+    char *string = ((char**)cp["my_strings"])[0];
+    int32_t number = ((int32_t*)cp["my_numbers"])[0];
+    
+//    char *string = ((char**)cp[0])[0];
+//    int32_t number = ((int32_t*)cp[1])[0];
+
+    Serial.print(String(row_index) + ". String = ");
+    Serial.println(string);
+    Serial.print(String(row_index) + ". Number = ");
+    Serial.println(number, DEC);
+    Serial.println();
+    row_index++;
+  }
+
+  Serial.println();
+  cp.print();
+}
+
+void loop() {
+
+}

library.properties

@@ -1,5 +1,5 @@
 name=CSV Parser
-version=1.0.1
+version=1.1.0
 author=Michal Borowski <[email protected]>
 maintainer=Michal Borowski <[email protected]>
 sentence=CSV Parser for Arduino.