Send file post request from the ESP32

Question

Is it impossible to send a file post request from the ESP32 using the SPIFFS file system? ESP32 freezes while sending.

Code example:

File file = SPIFFS.open("/Images/test.bmp", FILE_READ);

String start_request = ""; String end_request = "";
start_request = start_request +
                "\n--AaB03x\n" +
                "Content-Disposition: form-data; name=\"userfile\"; filename=\"CAM.bmp\"\n" +
                "Content-Transfer-Encoding: binary\n\n";
end_request = end_request + "\n--AaB03x--\n";
uint16_t full_length;
full_length = start_request.length() + file.size() + end_request.length();

WiFiClient client;
if (!client.connect("server_name", 80)) {
  Serial.println("Connected FILED!");
  return;
}

Serial.println("Connected ok!");
client.println("POST /Home/Index HTTP/1.1");
client.println("Host: example.com");
client.println("User-Agent: ESP32");
client.println("Content-Type: multipart/form-data; boundary=AaB03x");
client.print("Content-Length: "); client.println(full_length);
client.print(start_request);

while (file.available()){
  client.write(file.read());
}

Serial.println(">>><<<");
client.println(end_request);

Answer 1

One can send a POST request with data fields appended with binary data as shown in the code below. The presented code was made to make a POST request to a Dataverse repository, however, can be easily adapted to any other POST request. Finally, the code below is optimized for Tensilica's Xtensa LX6/7 microprocessors, vis-a-vis ESP32.

The full code can be found here:

https://github.com/aeonSolutions/openScienceResearch-Smart-DAQ-Device-able-to-Upload-Live-Experimental-Sensor-Data-to-a-Data-Repo

// *********************************************************
//            Upload Dataset to Harvard's Dataverse
// *********************************************************
void UploadToDataverse() {
  //Check WiFi connection status
  if(WiFi.status() != WL_CONNECTED){
    mserial.printStrln("WiFi Disconnected");
    if (connect2WIFInetowrk()){
      UploadToDataverse();
    }
  }
  // Start sending dataset file
  File datasetFile = FFat.open("/"+EXPERIMENTAL_DATA_FILENAME, FILE_READ);
  if (!datasetFile){
    mserial.printStrln("Dataset file not found");
    return;
  }
    
  String boundary = "7MA4YWxkTrZu0gW";
  String contentType = "text/csv";
  // DATASET_REPOSITORY_URL = "/api/files/:persistentId/replace?persistentId=" +PERSISTENT_ID;
  DATASET_REPOSITORY_URL =  "/api/datasets/:persistentId/add?persistentId="+PERSISTENT_ID;
  
  String datasetFileName = datasetFile.name();
  String datasetFileSize = String(datasetFile.size());
  mserial.printStrln("Dataset File Details:");
  mserial.printStrln("Filename:" + datasetFileName);
  mserial.printStrln("size (bytes): "+ datasetFileSize);
  mserial.printStrln("");
    
  WiFiClientSecure client;

  int str_len = SERVER_URL.length() + 1; // Length (with one extra character for the null terminator)
  char SERVER_URL_char [str_len];    // Prepare the character array (the buffer) 
  SERVER_URL.toCharArray(SERVER_URL_char, str_len);    // Copy it over 
    
  client.stop();
  client.setCACert(HARVARD_ROOT_CA_RSA_SHA1);
  if (!client.connect(SERVER_URL_char, SERVER_PORT)) {
      mserial.printStrln("Cloud server URL connection FAILED!");
      mserial.printStrln(SERVER_URL_char);
      int server_status = client.connected();
      mserial.printStrln("Server status code: " + String(server_status));
      return;
  }
  mserial.printStrln("Connected to the dataverse of Harvard University"); 
  mserial.printStrln("");
  // We now create a URI for the request
  mserial.printStr("Requesting URL: ");
    mserial.printStrln(DATASET_REPOSITORY_URL);

  // Make a HTTP request and add HTTP headers    
  // post header
  String postHeader = "POST " + DATASET_REPOSITORY_URL + " HTTP/1.1\r\n";
  postHeader += "Host: " + SERVER_URL + ":" + String(SERVER_PORT) + "\r\n";
  postHeader += "X-Dataverse-key: " + API_TOKEN + "\r\n";
  postHeader += "Content-Type: multipart/form-data; boundary=" + boundary + "\r\n";
  postHeader += "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
  postHeader += "Accept-Encoding: gzip,deflate\r\n";
  postHeader += "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n";
  postHeader += "User-Agent: AeonLabs LDAD Smart DAQ device\r\n";
  postHeader += "Keep-Alive: 300\r\n";
  postHeader += "Connection: keep-alive\r\n";
  postHeader += "Accept-Language: en-us\r\n";

  // jsonData header
  String jsonData = "{\"description\":\"LIVE Experimental data upload from LDAD Smart 12bit DAQ \",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}";     
  String jsonDataHeader = "--" + boundary + "\r\n";
  jsonDataHeader += "Content-Disposition: form-data; name=\"jsonData\"\r\n\r\n";
  jsonDataHeader += jsonData+"\r\n";

  // dataset header
  String datasetHead = "--" + boundary + "\r\n";
  datasetHead += "Content-Disposition: form-data; name=\"file\"; filename=\"" + datasetFileName + "\"\r\n";
  datasetHead += "Content-Type: " + contentType + "\r\n\r\n";

  // request tail
  String tail = "\r\n--" + boundary + "--\r\n\r\n";

  // content length
  int contentLength = jsonDataHeader.length() + datasetHead.length() + datasetFile.size() + tail.length();
  postHeader += "Content-Length: " + String(contentLength, DEC) + "\n\n";
  
  // send post header
  int postHeader_len=postHeader.length() + 1; 
  char charBuf0[postHeader_len];
  postHeader.toCharArray(charBuf0, postHeader_len);
  client.print(charBuf0);
  mserial.printStr(charBuf0);

  // send key header
  char charBufKey[jsonDataHeader.length() + 1];
  jsonDataHeader.toCharArray(charBufKey, jsonDataHeader.length() + 1);
  client.print(charBufKey);
  mserial.printStr(charBufKey);

  // send request buffer
  char charBuf1[datasetHead.length() + 1];
  datasetHead.toCharArray(charBuf1, datasetHead.length() + 1);
  client.print(charBuf1);
  mserial.printStr(charBuf1);

  // create buffer
  const int bufSize = 2048;
  byte clientBuf[bufSize];
  int clientCount = 0;

  while (datasetFile.available()) {
    clientBuf[clientCount] = datasetFile.read();
    clientCount++;
    if (clientCount > (bufSize - 1)) {
        client.write((const uint8_t *)clientBuf, bufSize);
        clientCount = 0;
    }
  }

  datasetFile.close();
    
  if (clientCount > 0) {
      client.write((const uint8_t *)clientBuf, clientCount);
      mserial.printStrln("[binary data]");
  }

  // send tail
  char charBuf3[tail.length() + 1];
    tail.toCharArray(charBuf3, tail.length() + 1);
    client.print(charBuf3);
  mserial.printStr(charBuf3);



  // Read all the lines of the reply from server and print them to mserial
    mserial.printStrln("");
    mserial.printStrln("Response Headers:");
  String responseHeaders = "";

  while (client.connected()) {
    // mserial.printStrln("while client connected");
    responseHeaders = client.readStringUntil('\n');
    mserial.printStrln(responseHeaders);
    if (responseHeaders == "\r") {
      mserial.printStrln("======   end of headers ======");
      break;
    }
  }

  String responseContent = client.readStringUntil('\n');
  mserial.printStrln("Harvard University's Dataverse reply was:");
  mserial.printStrln("==========");
  mserial.printStrln(responseContent);
  mserial.printStrln("==========");
  mserial.printStrln("closing connection");
  client.stop();
  }