Seeking to write a ton of information to an SD card as close to live as possible

Question

So I've recently started a project where I am using an accelerometer, along with a SD card breakout board. I've been able to get information to write properly to the SD card with no issues whatsoever. However, my speed is a bit lower than I would like, at about only 40Hz or so, where my accelerometer can log data at 260Hz (MPU-6050). Each reading contains an X, Y, Z, and timestamp value since recording started. Each reading is output into a .txt file for later separation via comma-separated-value sorting in Excel(I wish) or GNUplot, which handles the massive sample size much better.

I've referenced https://forum.arduino.cc/t/how-to-write-data-with-high-sampling-rates-to-a-sd-card/281496/9 to try and figure this out, but I couldn't get much out of it. I don't understand arrays very well beyond the fact they are just tables like in excel, and can be set up in 1, 2, 3, or more dimensions.

I've read around a bit and it seems my problem is that I'm closing and opening the file each and every write cycle. However, I don't know how to get out of doing this as the program seems to not run properly if I don't close out the file or if I use flush. My general idea of how this could flow without opening/close cycles every single time would be:

1. Create arrays for each variable (X, Y, Z, elapsedTime)
2. Take all my recordings for a given time, maybe five-hundred readings or so which would come out to about 12 seconds, or whatever possible value I can squeeze out of my Atmega328P and maintain program stability, and shove them into their respective arrays. Ideally the longer the better, but I understand I have memory constraints.
3. In one write-cycle, I write all the values into CSV form, or even multiple print cycles cycling through each array cell continuously until all the values are input with the proper format.
4. Clear the arrays of all the info they held, and return to step 2.

Really appreciate any help you guys can offer. Thank you!

//

#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
int recordStatus = false;                                                   //Default recordStatus is zero.
int recordingLed = 3; 
const int chipSelect = 10;
Adafruit_MPU6050 mpu;

void setup(void) {
  pinMode(2, INPUT_PULLUP);
  pinMode(3, OUTPUT);
  Serial.begin(500000);
  while (!Serial)
    delay(10); // will pause Zero, Leonardo, etc until serial console opens
                                                                              
                                                                           //SD CARD WRITE CODE BEGIN                                                                              
 {
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }

  Serial.print("Initializing SD card...");

  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    // don't do anything more:
    while (1);
  }
  Serial.println("card initialized.");
}
                                                                              //SD CARD WRITE CODE END  

  Serial.println("Adafruit MPU6050 test!");                                   //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN
  // Try to initialize!
  if (!mpu.begin()) {
    Serial.println("Failed to find MPU6050 chip");
    while (1) {
      delay(10);
    }
  }
  Serial.println("MPU6050 Found!");

  mpu.setAccelerometerRange(MPU6050_RANGE_8_G);
  switch (mpu.getAccelerometerRange()) {
  case MPU6050_RANGE_2_G:
    break;
  case MPU6050_RANGE_4_G:
    break;
  case MPU6050_RANGE_8_G:
    break;
  case MPU6050_RANGE_16_G:
    break;
  }
  mpu.setGyroRange(MPU6050_RANGE_500_DEG);
  switch (mpu.getGyroRange()) {
  case MPU6050_RANGE_250_DEG:
    break;
  case MPU6050_RANGE_500_DEG:
    break;
  case MPU6050_RANGE_1000_DEG:
    break;
  case MPU6050_RANGE_2000_DEG:
    break;
  }

  mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);
  Serial.print("Filter bandwidth set to: ");
  switch (mpu.getFilterBandwidth()) {
  case MPU6050_BAND_260_HZ:
    Serial.println("260 Hz");
    break;
  case MPU6050_BAND_184_HZ:
    Serial.println("184 Hz");
    break;
  case MPU6050_BAND_94_HZ:
    Serial.println("94 Hz");
    break;
  case MPU6050_BAND_44_HZ:
    Serial.println("44 Hz");
    break;
  case MPU6050_BAND_21_HZ:
    Serial.println("21 Hz");
    break;
  case MPU6050_BAND_10_HZ:
    Serial.println("10 Hz");
    break;
  case MPU6050_BAND_5_HZ:
    Serial.println("5 Hz");
    break;
  }
                                                                                    //MPU6050 ACCELEROMETER INITIALIZATION CODE END
}

unsigned long startMillis;
unsigned long currentMillis;
unsigned long elapsedTime;
int rawMillis;
void loop() {
  digitalWrite(recordingLed, LOW);
  int buttonRecord = digitalRead(2);                                                //Check button for GND input
  /* Get new sensor events with the readings */
  sensors_event_t a, g, temp;
  mpu.getEvent(&a, &g, &temp);

// make a string for assembling the data to log:                                    //SD CARD VALUES ASSIGN BEGIN
  int X = "";
  int Y = "";
  int Z = "";
  String timeofRecord = String(millis()/1000.0, 10);
  X = (a.acceleration.x/9.81);
  Y = (a.acceleration.y/9.81);
  Z = (a.acceleration.z/9.81);
  if(buttonRecord == false){                                                        //Toggle record status upon pin 2 going to GND
    delay(500);
    recordStatus = !recordStatus;                                                   //If button is pressed, make recordStatus switch from true to false.
    startMillis = millis();
  }

if(recordStatus == true){                                                           //While record is toggled, do below
    rawMillis = millis()-startMillis;
    String elapsedTime = String(rawMillis/1000.00, 3);
    File dataFile = SD.open("datalog.txt", FILE_WRITE);
      if (dataFile) {                                                               // if the file is available, write to it:
    digitalWrite(recordingLed, HIGH);
    dataFile.print(elapsedTime);
    dataFile.print(", ");
    dataFile.print(X);
    dataFile.print(", ");
    dataFile.print(Y);
    dataFile.print(", ");
    dataFile.print(Z);
    dataFile.println("");
    dataFile.close();
  }
}
  // if the file isn't open, pop up an error:
  else {
    Serial.println("error opening datalog.txt");
    }
}

UPDATE

So I've been messing with this a while, and I believe I just about have it figured out. I have arrays set up for time, X, Y, and Z and they are populating values correctly. However, I can't seem to get into my SD card file. There is no issue when the setup initializes, it's like it skips the

if (dataFile){

entirely. The modified code is below here. There's no shutdowns, and the program loops continuously so I don't think I'm running out of memory or having any power issues. The red LED on the SD card is flashing as though it's being written. Even if I remove the if(dataFile) check and force the commands contained in the if statement to execute, my txt file is still left empty. Previously I was writing to this just fine. Memory problem? I have less than 500 bytes of memory available. Possibly this is the cause? Or is it the for loop causing my trouble?

#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
int recordStatus = false;                                                   //Default recordStatus is zero.
int recordingLed = 3; 
const int chipSelect = 10;
Adafruit_MPU6050 mpu;

void setup(void) {
  pinMode(2, INPUT_PULLUP);
  pinMode(3, OUTPUT);
  Serial.begin(500000);
  while (!Serial)
    delay(10);                                                       
                                                                           //SD CARD WRITE CODE BEGIN                                                                              
 {
  Serial.print("Initializing SD card...");

  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    // don't do anything more:
    while (1);
  }
  Serial.println("card initialized.");
}
                                                                              //SD CARD WRITE CODE END  

  Serial.println("Adafruit MPU6050 test!");                                   //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN
  // Try to initialize!
  if (!mpu.begin()) {
    Serial.println("Failed to find MPU6050 chip");
    while (1) {
      delay(10);
    }
  }
  Serial.println("MPU6050 Found!");

  mpu.setAccelerometerRange(MPU6050_RANGE_4_G);
  switch (mpu.getAccelerometerRange()) {
  case MPU6050_RANGE_2_G:
    break;
  case MPU6050_RANGE_4_G:
    break;
  case MPU6050_RANGE_8_G:
    break;
  case MPU6050_RANGE_16_G:
    break;
  }
  mpu.setGyroRange(MPU6050_RANGE_500_DEG);
  switch (mpu.getGyroRange()) {
  case MPU6050_RANGE_250_DEG:
    break;
  case MPU6050_RANGE_500_DEG:
    break;
  case MPU6050_RANGE_1000_DEG:
    break;
  case MPU6050_RANGE_2000_DEG:
    break;
  }

  mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);
  Serial.print("Filter bandwidth set to: ");
  switch (mpu.getFilterBandwidth()) {
  case MPU6050_BAND_260_HZ:
    Serial.println("260 Hz");
    break;
  case MPU6050_BAND_184_HZ:
    Serial.println("184 Hz");
    break;
  case MPU6050_BAND_94_HZ:
    Serial.println("94 Hz");
    break;
  case MPU6050_BAND_44_HZ:
    Serial.println("44 Hz");
    break;
  case MPU6050_BAND_21_HZ:
    Serial.println("21 Hz");
    break;
  case MPU6050_BAND_10_HZ:
    Serial.println("10 Hz");
    break;
  case MPU6050_BAND_5_HZ:
    Serial.println("5 Hz");
    break;
  }
                                                                                    //MPU6050 ACCELEROMETER INITIALIZATION CODE END
}

float startMillis;
float elapsedTime[3];
float rawMillis;

float X[3];
float Y[3];
float Z[3];

void loop() {
  digitalWrite(recordingLed, LOW);
  int buttonRecord = digitalRead(2);                                                //Check button for GND input
  /* Get new sensor events with the readings */
  if(buttonRecord == false){                                                        //Toggle record status upon pin 2 being pulled down to GND
    delay(500);
    recordStatus = !recordStatus;                                                   //If button is pressed, make recordStatus switch from true to false.
    startMillis = millis();
    Serial.print("BUTTON PRESSED");
  }
  if(recordStatus == true){                                                         //While record is toggled, do below
   digitalWrite(recordingLed, HIGH);
    sensors_event_t a, g, temp;
    mpu.getEvent(&a, &g, &temp);
for(int arrayNumber = 0; arrayNumber <= 3; ++arrayNumber){
    sensors_event_t a, g, temp;
    mpu.getEvent(&a, &g, &temp);
    
    X[arrayNumber] = (a.acceleration.x);                                             //SD CARD VALUES ASSIGN BEGIN/DO MATH TO CALCULATE G'S
    Y[arrayNumber] = (a.acceleration.y);
    Z[arrayNumber] = (a.acceleration.z);
    
    rawMillis = millis()-startMillis;
    elapsedTime[arrayNumber] = rawMillis;
    Serial.println(elapsedTime[arrayNumber]);
    Serial.println(X[arrayNumber]);
    Serial.println(Y[arrayNumber]);
    Serial.println(Z[arrayNumber]);
    Serial.println(arrayNumber);                                                     // diagnostic for checking to make sure arrays are being populated properly
}                                                          

    File dataFile = SD.open("datalog.txt", FILE_WRITE);
    Serial.print("LEFT LOGGING LOOP, ENTERING WRITE LOOP");
    
  if (dataFile) {                                                                    // if the file is available, write to it
    for(int arrayNumber = 0; arrayNumber <= 3; ++arrayNumber){
 Serial.print("NOW INSIDE WRITE LOOP");                                              // confirmation of being inside the write loop, and writing data to the file
    String(dataString) = (
    elapsedTime[arrayNumber] + String(", " ) + X[arrayNumber] + String(", ") + Y[arrayNumber] + String(", ") + Z[arrayNumber]         //Format data for output to the comma separated format for excel/gnuplot
  );
    dataFile.println(dataString);
       }
       dataFile.close();
       Serial.println("WRITE LOOP DONE");
    }
  }
}

Answer 1

So I spent a while longer banging my head on the wall, and I finally found a solution to my issue. @DaveNewton provided several links that lead to other links, and I found my way to here where some not-so-common knowledge is explained. It appears that for some reason the FILE_WRITE function is slow, and takes a lot of time to execute.

This can be substituted by using

O_CREAT | O_WRITE | O_APPEND

as this first checks if the file is created, and if not, creates it. Once the or check passes for the O_CREAT section, it moves on into the O_WRITE command. Then the O_APPEND checks if there is any data already present and if so, move to the end of the line to begin new data logging.

My finished code in it's entirety is below, I hope it helps someone in the future. I'm using a MicroCenter 32GB (class 10) Micro SD card to store the data, this SD card breakout board, and this MPU-6050 based accelerometer.

// This program is designed to log data to an SD card via an arduino Uno, and SD card breakout board, and a MPU-6050 accelerometer module.

// MPU-6050:                https://www.amazon.com/HiLetgo-MPU-6050-Accelerometer-Gyroscope-Converter/dp/B078SS8NQV
// Micro SD Card Breakout:  https://www.adafruit.com/product/254
// Micro SD Card I used:    https://www.microcenter.com/product/485584/micro-center-32gb-microsdhc-card-class-10-flash-memory-card-with-adapter

// Link to original Arduino StackExchange submission where I worked through this code:  https://arduino.stackexchange.com/questions/84961/seeking-to-write-a-ton-of-information-to-an-sd-card-as-close-to-live-as-possible

#include <Adafruit_MPU6050.h>
#include <Adafruit_Sensor.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>

int recordStatus = false;                                                   //Default recordStatus is zero.
int recordingLed = 3;                                                       //Output to show us whether we are recording or not.
const int chipSelect = 10;                                                  //Use pin 10 as our CS pin
Adafruit_MPU6050 mpu;

void setup(void) {
  pinMode(2, INPUT_PULLUP);                                               //Set pin 2 to have a default HIGH state using the internal pull up resistor.
  pinMode(3, OUTPUT);
  Serial.begin(500000);
  SPI.begin();
  SPI.beginTransaction(SPISettings(16000000, MSBFIRST, SPI_MODE0));       //I have no idea if this really does anything, but I figure I'd just leave it since it's not causing any problems.
  while (!Serial)
    delay(10); // will pause Zero, Leonardo, etc until serial console opens                                                        
                                                                           //SD CARD WRITE CODE BEGIN                                                                              
 {

  Serial.print("Initializing SD card...");

  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    // don't do anything more:
    while (1);
  }
  Serial.println("card initialized.");
}
                                                                              //SD CARD WRITE CODE END  

  Serial.println("Adafruit MPU6050 test!");                                   //MPU6050 ACCELEROMETER INITIALIZATION CODE BEGIN
  // Try to initialize!
  if (!mpu.begin()) {
    Serial.println("Failed to find MPU6050 chip");
    while (1) {
      delay(10);
    }
  }
  Serial.println("MPU6050 Found!");

  mpu.setAccelerometerRange(MPU6050_RANGE_16_G);                            //I originally had this at 4 but was capping results, so I moved it up to 16. Feel free to lower this if you know your application will not exceed 4Gs
  switch (mpu.getAccelerometerRange()) {
  case MPU6050_RANGE_2_G:
    break;
  case MPU6050_RANGE_4_G:
    break;
  case MPU6050_RANGE_8_G:
    break;
  case MPU6050_RANGE_16_G:
    break;
  }
  mpu.setGyroRange(MPU6050_RANGE_500_DEG);                                //I'm not using the gyro at all, but 500 degrees seems like a good value to use here.
  switch (mpu.getGyroRange()) {
  case MPU6050_RANGE_250_DEG:
    break;
  case MPU6050_RANGE_500_DEG:
    break;
  case MPU6050_RANGE_1000_DEG:
    break;
  case MPU6050_RANGE_2000_DEG:
    break;
  }

  mpu.setFilterBandwidth(MPU6050_BAND_260_HZ);                              //Select maximum frequency for recording.
  Serial.print("Filter bandwidth set to: ");
  switch (mpu.getFilterBandwidth()) {
  case MPU6050_BAND_260_HZ:
    Serial.println("260 Hz");
    break;
  case MPU6050_BAND_184_HZ:
    Serial.println("184 Hz");
    break;
  case MPU6050_BAND_94_HZ:
    Serial.println("94 Hz");
    break;
  case MPU6050_BAND_44_HZ:
    Serial.println("44 Hz");
    break;
  case MPU6050_BAND_21_HZ:
    Serial.println("21 Hz");
    break;
  case MPU6050_BAND_10_HZ:
    Serial.println("10 Hz");
    break;
  case MPU6050_BAND_5_HZ:
    Serial.println("5 Hz");
    break;

   SPI.endTransaction();
  }
                                                                                    //MPU6050 ACCELEROMETER INITIALIZATION CODE END
}
float X;
float Y;
float Z;
float startMillis;
float currentMillis;
float rawMillis;
int tickMillis = LOW;

void loop() {
  int buttonRecord = digitalRead(2);                                                //Check button (pin 2) to see if it is grounded through momentary switch.
  // Get new sensor events with the readings.

if(buttonRecord == LOW){                                                           //If buttonRecord switches to GND, switch tickMillis to it's opposite value as a toggle and store startMillis, then wait 300ms as a debounce.
    tickMillis = !tickMillis;
    startMillis = millis();
    delay(300);
}
    if(tickMillis == HIGH){
    File dataFile = SD.open("datalog.txt", O_CREAT | O_WRITE | O_APPEND);           // Create datalog.txt. If already created, move to end of stored data, and begin write function
    if (dataFile) {                                                                 // if the file is available, write to it:

  for(uint8_t i = 0; i < 250; i++){                                                 // For i = 0, execute the below code. Then increment i by 1. Once past 250, exit for() loop
    digitalWrite(recordingLed, HIGH);

    sensors_event_t a, g, temp;                                                     // Commands to trigger event sensing and acceleration logging.
    mpu.getEvent(&a, &g, &temp);

    X = a.acceleration.x/9.81;                                                      // Divide normal acceleration return values to calculate G forces
    Y = a.acceleration.y/9.81;
    Z = a.acceleration.z/9.81;

    rawMillis = millis()-startMillis;                                               // Take our current millis value and subtract our beginning startMillis from it to get how long we've actually been recording.

    dataFile.print(rawMillis/1000, 3);                                              // Store data in a Comma-Seperated-Value format. The '3' after rawMillis/1000 says print out to 3 decimal places.
    dataFile.print(", ");                                                           // I just use a txt file since GNUplot can plot from that value, but a CSV library could probably be used here.
    dataFile.print(X);                                                              // Excel doesn't allow plots beyond 255 datapoints for god knows why. So GNUplot is basically a necessity to plot these accurately to timestamp.
    dataFile.print(", ");
    dataFile.print(Y);
    dataFile.print(", ");
    dataFile.print(Z);
    dataFile.println();
    }
    dataFile.flush();
    dataFile.close();
    }
   else {
    delay(500);
    Serial.println("error opening datalog.txt");
    }
  } else {
        digitalWrite(recordingLed, LOW);
    }
}

Thanks so much for the comments/advice everyone. It's really appreciated. The final recording frequency for this is ~170Hz, which is good enough for me.