File Size Problem with EyeFi SD Card and Arduino Mega

Question

I've been writing to an EyeFi SD card with an Arduino Mega. I've managed to get the files to upload wirelessly, but I noticed that my data gets cut off when the file reaches a size of 24 kb. If I use a normal SD card, the file is able to keep logging past 24 kb. The EyeFi card is able to upload large images, so I'm not sure what is limiting my file size for the EyeFi card. Can I get some advice?

/*************** SETUP ***************/
void setup(void)
{
  Serial.begin(115200);

  // initialize the SD card
  Serial.print("Initializing SD card...");
  pinMode(10, OUTPUT); // make sure that the default chip select pin is set to

  // see if the card is present and can be initialized:
  if (!SD.begin(10,11,12,13))
  {
    error("Card failed, or not present");
  }
  Serial.println("Card initialized");

  // see if the directory exists, create it if not.
  if( !SD.exists(directory) && !SD.mkdir(directory) )
    error("File directory not created");    

  newFile();

  Serial1.begin(115200);

  while (Serial1.peek() != '!' )
    Serial1.read();

  i = 0;  
}

/*************** MAIN LOOP ***************/
void loop(void)
{
  if (Serial1.available())
  {
    logfile.print( (char)Serial1.read() );
    i++;
  }

  if (i >= SOFT_BUFFER_SIZE)
  {
    logfile.flush();

    if( logfile.size() > 20480 && logfile.size() < (20480 + SOFT_BUFFER_SIZE) )
      Serial.println("Minimum File Size Achieved.");

    i = 0;
  }
}

Answer 1

After some testing and research, I found out that the Eye-Fi SD card starts a transfer once a file reaches 20kb or more and will not upload a file again once it has been transfered already. The solution, as suggested by ChrisStratton, is to log to a temporary file and trigger a transfer once the file is ready.

I took advantage of the fact that the Eye-Fi only transferring files in the standard camera folders. In my case, I used the Nikon naming convention: DCIM/100NIKON/DSCNxxxx.JPG (xxxx is the photo number ranging from 0000 to 9999), but there are also others. Check on the Eye-Fi website for compatible cameras. I created a temporary file called TEMP.TMP and stored it to the root folder of the card and logged to it. Once I'm ready to transfer the file, TEMP.TMP gets renamed to "DCIM/100NIKON/DSCNxxxx.JPG" and the whole file is transferred all at once.

Here's my code. I'm using the software serial (pins 10, 11, 12, 13) of the Mega:

//Running with 256kb serial buffer (modified HardwareSerial.cpp)
//Added file time stamps

#include "SdFat.h"
#include "SdFatConfig.h"
#include <Wire.h>
#include "RTClib.h"

#define SYNC_INTERVAL 1000 // mills between calls to flush() - to write data to the card
uint32_t syncTime = 0; // time of last sync()

#define ECHO_TO_Serial     1 // echo data to Serial port
#define WAIT_TO_START      0 // Wait for Serial input in setup()
#define MESSAGE_SIZE      11 // Message size of "!G N +1234" + CR
#define SOFT_BUFFER_SIZE  256// 25 Messages of 11 chars each

const uint8_t chipSelect = 10;

SdFat sd;
SdFile logfile; //create log file

RTC_DS1307 RTC;
DateTime filedate;

uint16_t date;
uint16_t time;

int i = 0;
int x = 0;

char directory[] = "DCIM/100NIKON";
char filename[] = "DCIM/100NIKON/DSCN0000.JPG";
char tempfile[] = "TEMP.TMP";

/*************** ERROR ***************/
void error(char *str)
{
  Serial.print("error: ");
  Serial.println(str);
  while(1);
}

/*********** TEMP FILE CREATION ***********/
void newFileName()
{
  sd.chdir("/"); //check DCIM/100NIKON directory
  // create a temp file 
  for (uint16_t i = 0; i < 10000; i++)
  {
    filename[18] = i/1000 + '0';
    filename[19] = (i/100)%10 + '0';
    filename[20] = (i/10)%10 + '0';
    filename[21] = i%10 + '0';
    if (! sd.exists(filename))
    {
      Serial.print("New file name: ");
      Serial.println(filename);
      break;  // leave the loop!
    }
  }
}

/****** call back for file timestamps *****/
void dateTime(uint16_t* date, uint16_t* time) {
  DateTime now = RTC.now();

  // return date using FAT_DATE macro to format fields
  *date = FAT_DATE(now.year(), now.month(), now.day());

  // return time using FAT_TIME macro to format fields
  *time = FAT_TIME(now.hour(), now.minute(), now.second());
}

/*************** SETUP ***************/
void setup(void)
{
  Serial.begin(115200);
  Wire.begin();
  RTC.begin();

  if(! RTC.isrunning() )
    Serial.println("RTC is not running");

  // set date time callback function
  SdFile::dateTimeCallback(dateTime); 

  // initialize the SD card
  Serial.print("Initializing SD card...");

  // see if the card is present and can be initialized:
  if (! sd.begin(chipSelect, SPI_HALF_SPEED))
    error("Card failed, or not present");

  Serial.println("Card initialized");

  // see if the directory exists, create it if not.
  sd.chdir("/");
  if( !sd.exists(directory) && !sd.mkdir(directory) )
    error("File directory not created"); 

  // create new file name
  newFileName();

  // Scan root folder for temp files
  sd.chdir("/");
  while( logfile.openNext(sd.vwd(), O_READ)){
    if( !logfile.isDir() ) {     
      logfile.printName(&Serial);
      if( logfile.fileSize() < 20480 ){
        Serial.print(" file size too small. ");
        Serial.print(logfile.fileSize());
        Serial.println(" bytes.");
        logfile.remove(); //Delete under sized files
      }else
        if( !logfile.rename(sd.vwd(), filename))
          error(" could not be renamed.");
        else{
          Serial.print(" renamed to ");
          Serial.println(filename);
        }
    }
    logfile.close();
  }

  // create temp file
  sd.chdir("/");
  if(!logfile.open(tempfile, O_WRITE | O_CREAT)) // only open a new file if it doesn't exist
    error("couldnt create temp file");

  Serial1.begin(115200);

  // Rotate the buffer
  for( int j = 0; j < 256; j++) {
    Serial1.read();
  }

  // Look for the '!' character
  while (Serial1.peek() != '!' )
    Serial1.read();

  Serial.println("Logging...");
  Serial.println();

  // Reset buffer index to 0
  i = 0;  
}

/*************** MAIN LOOP ***************/
void loop(void)
{
  if (Serial1.available()){
    logfile.print( (char)Serial1.read() );  
    i++;
  }

  if (i >= SOFT_BUFFER_SIZE)
  {
    logfile.sync();
    i = 0;
  }
}