Writing to sd card with a sampling rate 50ms or less? I have a sketch that writes to sd every 1s but jams at faster sampling rates. Please help?

Question

My code below reads data from an ADC and 3 digital pins via a Mega and prints to an SD Shield (Deek Robot). I want to be able to read the data every 50ms (20ms ideally). I can't get below 250ms.

I've read countless forum posts regarding the same issue and understand that I need to create 2x 512 byte buffer, store data to one while the other is being written to the SD card. A forum post mentioned using the Adafruit GPS library for an example of double-buffering but I can't seem to find it. Another suggests changing the internal buffer from 64bytes to 256bytes in hardwareserial.cpp but I can't even navigate to it on my w***ows laptop.

I've run dataString.Length and the max length is 38 for the 8 columns of data, plus a long (for recording ms) another byte for the "\t" and 3 bytes of headroom = 46bytes in all, round it up to 50bytes/sample at a 50ms sampling rate is 1000bytes/second. Darn it, I forgot to add the separating-commas between the columns = plus another 7 bytes; call it 55 in total = 1100bytes/second. I know it can be done but I can't do it; not at my current knowledge/skill level.

Thank you in advance for your help.

//include relevant libraries 
#include <EnableInterrupt.h>
#include <SPI.h>
#include <SD.h>

//globally define pins
#define BUSY 3    //purple*
#define RD 4      //yellow* RD+CS tied together
#define RESET 5   //grey* 
#define CONVST 6  //green* CONVSTA+CONVSTB soldered together on the board
#define RANGE 7   //blue*   *not permanent - double check!!!*

#define DB0 22
#define DB1 23
#define DB2 24
#define DB3 25
#define DB4 26
#define DB5 27
#define DB6 28
#define DB7 29
#define DB8 30
#define DB9 31
#define DB10 32
#define DB11 33
#define DB12 34
#define DB13 35
#define DB14 36
#define DB15 37

//byte statusLed       = 13;
byte sensorPin1       = 38;
byte sensorPin2       = 40;
byte sensorPin3       = 42;

int sensorValue[8];
int rawData[16];
//change the size  of these arrays to match the amount of channels being read by the ADC 
int adcChannel[5];
int adcData[5];
int channelCount=5;

volatile byte pulseCount1;
volatile byte pulseCount2;
volatile byte pulseCount3;

int pulses[3];

// cs pin for sd-shield *NB 53 for Mega
const int chipSelect = 53;
// SCK 52
// MISO 50
// MOSI 11

unsigned long oldTime;

void setup() {             //set up the hardware

  Serial.begin(9600);     
  //sd card.....  
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }

  Serial.print("Initializing SD card...");

  // see if the card is present and can be initialized:
  if (!SD.begin(chipSelect)) {
    Serial.println("Card failed, or not present");
    // don't do anything more:
    while (1);
  }
  Serial.println("card initialized.");

  enableInterrupt(BUSY, bitBang, FALLING);

  oldTime            = 0;

  pinMode(DB0, INPUT_PULLUP);
  pinMode(DB1, INPUT_PULLUP);
  pinMode(DB2, INPUT_PULLUP);
  pinMode(DB3, INPUT_PULLUP);
  pinMode(DB4, INPUT_PULLUP);
  pinMode(DB5, INPUT_PULLUP);
  pinMode(DB6, INPUT_PULLUP);
  pinMode(DB7, INPUT_PULLUP);
  pinMode(DB8, INPUT_PULLUP);
  pinMode(DB9, INPUT_PULLUP);
  pinMode(DB10, INPUT_PULLUP);
  pinMode(DB11, INPUT_PULLUP);
  pinMode(DB12, INPUT_PULLUP);
  pinMode(DB13, INPUT_PULLUP);
  pinMode(DB14, INPUT_PULLUP);
  pinMode(DB15, INPUT_PULLUP);

  pinMode(RESET, OUTPUT);
  pinMode(CONVST, OUTPUT);
  pinMode(RD, OUTPUT);
  pinMode(RANGE, OUTPUT);
  pinMode(BUSY, INPUT);

  //reset ADC to begin conversion
  digitalWrite(RESET, HIGH);
  delayMicroseconds(10);
  digitalWrite(RESET, LOW);

  digitalWrite(CONVST, LOW);
  digitalWrite(RD, HIGH);
  digitalWrite(RANGE, LOW);
  digitalWrite(BUSY, LOW);

  delayMicroseconds(100);

  // Set up the status LED line as an output
//  pinMode(statusLed, OUTPUT);
//  digitalWrite(statusLed, HIGH);  // We have an active-low LED attached

  pinMode(sensorPin1, INPUT);
  digitalWrite(sensorPin1, HIGH);

  pinMode(sensorPin2, INPUT_PULLUP);
  digitalWrite(sensorPin2, HIGH);

  pinMode(sensorPin3, INPUT_PULLUP);
  digitalWrite(sensorPin3, HIGH);

  pulseCount1        = 0;
  pulseCount2        = 0;
  pulseCount3        = 0;

  oldTime            = 0;

  // the hall effect sensors are configured to trigger on a FALLING state change
  // (transition from HIGH state to LOW state)
  enableInterrupt(sensorPin1, pulseCounter1, FALLING);
  enableInterrupt(sensorPin2, pulseCounter2, FALLING);
  enableInterrupt(sensorPin3, pulseCounter3, FALLING);

}

void loop() {

  //change this value to preffered sampling rate
  if ((millis() - oldTime) == 50) {

    oldTime = millis();

    //Serial.print(millis());
    //Serial.print("\t");

    //tell ADC to start read (converting analogue input to digital output)
    delayMicroseconds(10);
    digitalWrite(CONVST, LOW);
    delayMicroseconds(10);
    digitalWrite(CONVST, HIGH);
    //when read-conversion is complete ADC sends the BUSY pin low triggering the bitBang ISR

    //print ADC data from array within the bitBang ISR to a string
    String adcString = "";
    for(int thisChannel=0; thisChannel<channelCount; thisChannel++){
      adcString += String(adcData[thisChannel]);
      adcString += ",";
    }

    //disable interrupt to access current pulse count
    disableInterrupt(sensorPin1);
    disableInterrupt(sensorPin2);
    disableInterrupt(sensorPin3);
    //grab current pulse count and store in a variable
    pulses[0] = pulseCount1;
    pulses[1] = pulseCount2;
    pulses[2] = pulseCount3;
    //reset pulse count
    pulseCount1 = 0;
    pulseCount2 = 0;
    pulseCount3 = 0;
    //enable interrupt and start incrementing pulse count again
    enableInterrupt(sensorPin1, pulseCounter1, FALLING);
    enableInterrupt(sensorPin2, pulseCounter2, FALLING);
    enableInterrupt(sensorPin3, pulseCounter3, FALLING);

    String pulseString = "";
    for(int i=0; i<3; i++){
      pulseString += String(pulses[i]);
      if (i<2) {
        pulseString += ",";
      }
    }

   String dataString = String(adcString + pulseString);
  // open the file. note that only one file can be open at a time,
  // so you have to close this one before opening another.
  File dataFile = SD.open("dataLog.txt", FILE_WRITE);

  // if the file is available, write to it:
  if (dataFile) {
    dataFile.print(millis());
    dataFile.print(",");
    dataFile.println(dataString);
    dataFile.close();
    // print to the serial port too:
    //Serial.println(dataString);
  }  
  // if the file isn't open, pop up an error:
  else {
    Serial.println("error opening dataLog.txt");
  } 

 //Serial.println(dataString);

  }  
//start again
}

//grabbing ADC data ISR (for n channel)
void bitBang ()  {
  //a for loop to bitbang the values fom the analogue (ADC) channels (maximum 8) sequentially storing them in a variable array (equal to the channels of the ADC)
  for(int thisChannel=0; thisChannel<channelCount; thisChannel++){
  //send ADC read pin low to bitbang the first channel  
  digitalWrite(RD, LOW);
  //read the state of the 16 pins and store in a variable
  rawData[0] = digitalRead(DB15);
  rawData[1] = digitalRead(DB14);
  rawData[2] = digitalRead(DB13);
  rawData[3] = digitalRead(DB12);
  rawData[4] = digitalRead(DB11);
  rawData[5] = digitalRead(DB10);
  rawData[6] = digitalRead(DB9);
  rawData[7] = digitalRead(DB8);
  rawData[8] = digitalRead(DB7);
  rawData[9] = digitalRead(DB6);
  rawData[10] = digitalRead(DB5);
  rawData[11] = digitalRead(DB4);
  rawData[12] = digitalRead(DB3);
  rawData[13] = digitalRead(DB2);
  rawData[14] = digitalRead(DB1);
  rawData[15] = digitalRead(DB0);
  //convert into 16bit 2s compliment and store in a variable array 
  adcData[thisChannel] = rawData[0] | (rawData[1] << 1) | (rawData[2] << 2) | (rawData[3] << 3) | (rawData[4] << 4) | (rawData[5] << 5) | (rawData[6] << 6) | (rawData[7] << 7) |  (rawData[8] << 8) | (rawData[9] << 9) | (rawData[10] << 10) | (rawData[11] << 11) | (rawData[12] << 12) | (rawData[13] << 13) | (rawData[14] << 14) | (rawData[15] << 15);
  //send ADC pin high to say we have read the first channel 
  digitalWrite(RD, HIGH);
  //repeat for n channels
  }
}

//counting pulses ISRs
void pulseCounter1(){
  // Increment the pulse counter
  pulseCount1++;
}
void pulseCounter2(){
  // Increment the pulse counter
  pulseCount2++;
}
void pulseCounter3(){
  // Increment the pulse counter
  pulseCount3++;
}

Answer 1

Some tips:

• Do not open the open and close the file in every loop sequence (I think you can use the flush command to save/update the file.
• Do not save strings, but save the raw data and pulse string. This will take 16 * 2 (raw data + 3 bytes for the pulse data = 35 bytes per 16 samples, meaning 35 bytes/samples * 20 samples/s = 700 bytes (I think your calculation of 1100 bytes is incorrect).
• Try to avoid string, and even more string concatenation. I don't know if this will be the bottleneck, but in any case String operations are expensive and for concatenation there is a chance (or is always the case) that memory needs to be allocated dynamically. Use a string buffer instead.

Answer 2

So it turns out the actual problem was this line of code if ((millis() - oldTime) == 50) { it needed to be changed to if ((millis() - oldTime >= 50) {

I really appreciate all the help everyone gave me to get to the bottom of it and especially thankful to @Michel for handing some solid advice to someone who isn't in the know. Hopefully I will be able to repay in kind to another learner one day. :D