0

I'm a web developer and I started building a small environment logger that will post to my website`s API. I'm really new to this kind of coding so bare with me.

Since yesterday, the whole Arduino started to acting funny. After the first or second request to the server(with already tested code), I started sending corrupted JSON data.

{
  ":12": "module3",
  "10-7 18:26": {
    "1": {
      "": 21.50,
      "26:12": 55.50,
      "measured_at": "2015-10-7 18:26:12"
    },
    "2": {
      "": 21.10,
      "26:12": 55.70,
      "measured_at": "2015-10-7 18:26:12"
    }
  }

Where it should be

{
  "modulename": "module3",
  "environments": {
    "1": {
      "temperature": 21.40,
      "humidity": 55.60,
      "measured_at": "2015-10-7 18:25:52"
    },
    "2": {
      "temperature": 21.20,
      "humidity": 55.80,
      "measured_at": "2015-10-7 18:25:52"
    }
  }
}

The second JSON is from the initial request to the server. The first one is the corrupted one. I tried stopping all kinds of parts from the code, without any luck, except when I enter the measured_at manually (without getting the data from the RTC.)

#include <DHT.h>

#include <elapsedMillis.h>

#include <DS3231.h>
#include <Wire.h>
//Json Library
#include <ArduinoJson.h>

#include <Ethernet.h>
#include <SPI.h>

#include <LiquidCrystal.h>

byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED }; // RESERVED MAC ADDRESS
EthernetClient client;

const String module_name = "module3";

DHT dht1;
DHT dht2;

DS3231 clock;
RTCDateTime dt;

// initialize the library with the numbers of the interface pins

LiquidCrystal lcd(8, 9, 4, 5, 6, 7);

elapsedMillis timer0;
const long interval = 10000; // READING INTERVAL

int connected_sensors = 0;
float t1 = 0;  // TEMPERATURE VAR
float h1 = 0; // HUMIDITY VAR
float t2 = 0;
float h2 = 0;

int nc = 0; //Network connection simulation INI


byte temp[8] = { //icon for thermometer

  B00100,
  B01010,
  B01010,
  B01110,
  B01110,
  B11111,
  B11111,
  B01110
};

byte humi[8] = { //icon for water droplet
  B00100,
  B00100,
  B01010,
  B01010,
  B10001,
  B10001,
  B10001,
  B01110,
};

byte sconnect[8] = { //icon for connection
  0b00000,
  0b11111,
  0b10001,
  0b10001,
  0b11111,
  0b00100,
  0b11111,
  0b00000
};

void setup() {
  Serial.begin(9600);
  clock.begin();
  // set up the LCD's number of columns and rows:
  lcd.begin(16, 2);
  Serial.println("Terrarium Serial");
  // Print a message to the LCD.
  lcd.print("Weather v0.1");
  delay(1000);
  lcd.setCursor(0, 1);
  lcd.print("Sensor loading...");

  lcd.clear();
  lcd.print("Network check...");

  if (Ethernet.begin(mac) == 0) {
    Serial.println("Failed to configure Ethernet using DHCP"); //There is a problem with DHCP
    lcd.clear();
    lcd.setCursor(0, 0);
    lcd.print("DHCP ERROR");
    lcd.setCursor(0, 1);
    lcd.print("Please Restart");
    for (;;)
      ;
  }
  nc = 1;

  delay(2000);
  lcd.clear();
  lcd.setCursor(0, 0);
  lcd.print(Ethernet.localIP());
  lcd.setCursor(0, 1);
  lcd.println("Ready!");


  Serial.println(F("Connected!"));
  Serial.println(Ethernet.localIP());

  delay(4000);
  lcd.clear();

  lcd.createChar(1, temp);
  lcd.createChar(2, humi);
  lcd.createChar(3, sconnect);

  dht1.setup(15); 
  dht2.setup(16);

  h1 = dht1.getHumidity();
  t1 = dht1.getTemperature();

  h2 = dht2.getHumidity();
  t2 = dht2.getTemperature();

  delay(2000);

  //Display the statics on the display
  lcd.setCursor(2, 0);
  lcd.print(module_name);
  lcd.setCursor(9, 1);
  lcd.write(2);
  lcd.setCursor(15, 1);
  lcd.print("%");
  lcd.setCursor(2, 1);
  lcd.write(1);
  lcd.setCursor(8, 1);
  lcd.print((char)223);

  timer0 = 0; // clear the timer at the end of startup

}
void CheckSensors() {

  if (timer0 >= interval) { // READ ONLY ONCE PER INTERVAL
    timer0 -= interval;

    dt = clock.getDateTime(); //get current time

    h1 = dht1.getHumidity();
    t1 = dht1.getTemperature();

    h2 = dht2.getHumidity();
    t2 = dht2.getTemperature();

    DataSend(); //send data
  }
}

//Print the changeable values on the display
void LcdPint() {
  lcd.setCursor(0, 0);
  if (nc == 1) {
    lcd.write(3);
  } else if (nc == 0) {
    lcd.print(" ");
  } else if (nc == 2) {
    lcd.print("E");
  }

  lcd.setCursor(10, 1);
  lcd.print(h1);

  lcd.setCursor(3, 1);
  lcd.print(t1);

}

//Construct the JSON response
JsonObject& prepareResponse(JsonBuffer& jsonBuffer, String measured_at) {

  JsonObject& root = jsonBuffer.createObject();
  root["modulename"] = module_name;

  JsonObject& environmentsObject = root.createNestedObject("environments");


    JsonObject& environmentsOne = environmentsObject.createNestedObject("1"); 
    environmentsOne["temperature"] = t1;
    environmentsOne["humidity"] = h1;
    environmentsOne["measured_at"] =  measured_at;

    JsonObject& environmentsTwo = environmentsObject.createNestedObject("2");
    environmentsTwo["temperature"] = t2;
    environmentsTwo["humidity"] = h2;
    environmentsTwo["measured_at"] =  measured_at;

  return root;
}

void writeResponse(EthernetClient& client, JsonObject& json) {

  char buffer[256];
  json.printTo(buffer, sizeof(buffer));

  if (client.connect("api.terrariums.eu", 80)) { // REPLACE WITH YOUR SERVER ADDRESS
    client.println("POST /v1/sensors HTTP/1.1");
    client.println("Host: api.terrariums.eu"); // SERVER ADDRESS HERE TOO
    client.println("Content-Type: application/json");
    client.println("Authorization: Basic bW9kdWxlMzoxMzIxMzI=");
    client.println("Connection: close");
    client.print("Content-Length: ");
    client.println(strlen(buffer));
    client.println();
    client.println(buffer);

    nc = 1;
  } else {
    nc = 2;
    Serial.println("connection failed");
  }

  while (client.connected())
  {
    if ( client.available() )
    {
      char c = client.read();
      Serial.print(c);
    }
  }
  Serial.println();
  client.stop();
  //End  post request

}
void DataSend() {

    //This is the part that I think crashes my entire code. I tried using the helper with the RTC library that I use
    //but that made things worse.

    String measured_at = (String) dt.year + "-" + dt.month + "-" + dt.day  + " " + dt.hour   + ":" + dt.minute + ":" + dt.second;

    StaticJsonBuffer<200> jsonBuffer;
    JsonObject& json = prepareResponse(jsonBuffer, measured_at);
    json.prettyPrintTo(Serial);

    writeResponse(client, json);

}

void loop() {
  CheckSensors();
  LcdPint();
}
13
  • It even restarts the whole Arduino. I think this may be a conflict of some sort... I wonder why it shows up now, when I used it for 2 days straight a few weeks ago. Oct 7 '15 at 16:30
  • Chances are you're running out or memory. All that String use certainly doesn't help - get rid of those and things can only improve.
    – Majenko
    Oct 7 '15 at 16:37
  • This line, for instance: String measured_at = (String) dt.year + "-" + dt.month + "-" + dt.day + " " + dt.hour + ":" + dt.minute + ":" + dt.second; will be making swiss cheese of your heap.
    – Majenko
    Oct 7 '15 at 16:37
  • I would suggest ditching the JSON library as well, and just building the output up a bit at a time using client.print() - you don't have to send it all in one string, just do it a bit (even a character sometimes) at a time.
    – Majenko
    Oct 7 '15 at 16:41
  • Please tell me how you managed to format the code properly... I tried with that <-- language --> thing and i got nothing. Also I tried removing the measured_at and added another library, but that didnt help much either ... rinkydinkelectronics.com/library.php?id=73 This one i added and still... I`m a web dev, I write Javascript, this is really frustrating :D Could anyone show me a tutorial for proper ways to write for the Arduino .. I dont know.. I dont want to beg for someone to do it for me... Oct 7 '15 at 17:02
2

You have used 461 out of 2048 of your RAM bytes for string literals alone. At the very least, use the F() macro where you can, eg.

client.println(F("POST /v1/sensors HTTP/1.1"));
client.println(F("Host: api.terrariums.eu")); // SERVER ADDRESS HERE TOO
client.println(F("Content-Type: application/json"));
client.println(F("Authorization: Basic bW9kdWxlMzoxMzIxMzI="));
client.println(F("Connection: close"));
client.print(F("Content-Length: "));

Ditto for Serial.print and lcd.print where you are printing strings.

After that, what Majenko suggested is a good idea. Using a custom library is well and good, but JSON is easy enough to output directly. Just make a small function to do it for you.


Problem now is the Content-length... How do I calculate it ?

Do you absolutely need it? If you do, instead of building up the entire content first, I suggest two passes.

First pass outputs to a dummy class which simply calculates the length, second pass actually outputs. Something like this:

// outputting class that just counts output
class DummyOutput : public Print
{
private:
  unsigned long length_;

public:
  void begin () 
    { 
    length_ = 0; 
    }
  virtual size_t write (byte c)
    {
    length_++;
    return 1;
    }
  unsigned long getLength () const { return length_; }

};  // end of DummyOutput

DummyOutput countOutput;  // instance of DummyOutput

void setup ()
  {
  Serial.begin (115200);
  Serial.println ();
  Serial.println ("Starting");

  Print * outputter;  // where to write to

  for (int pass = 1; pass <= 2; pass++)
    {
    randomSeed (1);
    switch (pass)
     {
     case 1: outputter = &countOutput;
             countOutput.begin ();
             break;

     case 2: outputter = &Serial;
             break;
     } // end of switch

    // generate some output
    for (int i = 0; i < 25; i++)
      {
      outputter->print (random (5000));
      outputter->print ('\n');
      }

    switch (pass)
     {
     case 1: Serial.print (F("Content-length: "));
             Serial.println (countOutput.getLength ());
             break;

     case 2: break;
     } // end of switch

    }  // end of for loop
  }  // end of setup

void loop ()
  {
  }  // end of loop

Output:

Starting
Content-length: 121
1807
249
73
3658
3930
1272
2544
878
2923
2709
4440
3165
4492
3042
2987
2503
2327
1729
3840
2612
4303
3169
2709
2157
4560

By generating the same output in two passes you can first just calculate the length (without outputting) and the second time you can output that length first, and then output the actual data.

2
  • I already did the json with print only, as ugly as it is it works. Problem now is the Content-length... How do I calculate it ? Oct 7 '15 at 20:24
  • See amended reply.
    – Nick Gammon
    Oct 7 '15 at 22:08
1

In all probability you are either running out of memory, or the libraries you are using (and the fact that you are abusing String objects) is fragmenting your heap to such an extent that your whole 2KB of RAM is being used and you are ending up with a stack-and-heap collision.

You have to remember that the Arduino Uno only has 2KB of RAM. It's not a computer, and has no OS, and as such you have to unlearn much of what you know about programming and do things slightly differently.

The first problem you have is with your String abuse. This line:

String measured_at = (String) dt.year + "-" + dt.month + "-" + dt.day  + " " + dt.hour   + ":" + dt.minute + ":" + dt.second;

will be doing horrible things to your heap. The String class dynamically allocates memory on the heap to store the string content. The operations you have there inadvertently create a rather alarming number of temporary Strings which are then thrown away at the end. Every temporary String dynamically allocates RAM on the heap and can potentially lead to heap fragmentation.

For instance the numeric value dt.second is first converted to a String and memory allocated to store the content. Every concatenation results in a new String with its dynamic allocation, etc. Quite messy really.

Secondly, who knows what dynamic memory the ArduinoJSON library is using - quite a lot would be my guess - every tag, every value, etc, would need to be stored in memory somewhere, and unless they statically allocate a huge chunk of RAM (wasting most of your precious resource) they will be doing dynamic allocation - and that will make matters much worse.

So how would you get around it? It's actually quite simple - you need to think more about spoon-feeding a baby instead of preparing a 5 course banquet.

Instead of building up big objects with lots of data in them and sending them all out in one big chunk you need to just send little bits at a time. There is absolutely no reason to use things like ArduinoJSON if all you are doing is sending data.

For instance to send the JSON data you could do something like this:

  if (client.connect("api.terrariums.eu", 80)) { // REPLACE WITH YOUR SERVER ADDRESS
    client.println("POST /v1/sensors HTTP/1.1");
    client.println("Host: api.terrariums.eu"); // SERVER ADDRESS HERE TOO
    client.println("Content-Type: application/json");
    client.println("Authorization: Basic bW9kdWxlMzoxMzIxMzI=");
    client.println("Connection: close");
    client.println();

    client.println("{");
    client.println("    \"modulename\": \"module3\",");
    client.println("    \"environments\": {");
    client.println("        \"1\": {");
    client.print("            \"temperature\":
    client.print(t1);
    client.println(",");
    client.print("            \"humidity\":
    client.print(h1);
    client.println(",");
    client.println("        },");

Etc. You get the drift.

You notice I removed the Content-length header? That's because you don't know how long the content will be now (one drawback), so to signal the end of the transmission you will have to close the connection (I don't recall if client has a way of only half closing - just the TX portion) to signal to the web server that you have finished. However, if you know how long your numbers will be (say you format them to a specific number of characters) then your body size would always be the same and you can hard code a Content-length header.

Also - do you really need to send the time and date? Surely the computer has a better idea of what the current time and date is than the Arduino - just use the computer's internal clock.

7
  • Really nice, thank you. I decided that it was a good idea to have the date as it will store when the measurements were taken. Ofc I can use the moment when the code enters my API but I thought this wouldnt be a problem at that time. Oct 7 '15 at 17:36
  • Use the same trick for sending the date and time then. Instead of pre-formatting it as a String, just send each individual entry with a separate client.print().
    – Majenko
    Oct 7 '15 at 17:39
  • Ok. Will keep you updated. Oct 7 '15 at 17:40
  • How should I close the connection when I cant send a Content-length header? Oct 7 '15 at 20:05
  • Client.stop does that.
    – Majenko
    Oct 7 '15 at 20:52

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.