Sending data to Arduino Via Nodejs: Arduino does not echo back the data

Question

I have the following sketch that runs in a arduino UNO:

/*
 * Copyright 2018 Dimitrios Desyllas
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, 
 * including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 
 * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

String inputString = "";         // a String to hold incoming data
bool seralReadEnded=false;


void setup() {
  Serial.begin(19200);
  while (!Serial);
}

void loop() {
  if(seralReadEnded){
    sendDataOverSerial((byte [])inputString.c_str(),inputString.length());
    inputString="";
    seralReadEnded=false;
    delay(5000);
  }
}

String concatHex(byte value,String string){
  string+="\\x";
  if(value<16){
   string+="0";
  }
  string+=String(value,HEX);
  return string;
}

void sendDataOverSerial(byte byteArray[],size_t arrayLength){
  String returnVal="BEGIN:";
  for(int i=0;i<arrayLength;i++){
    returnVal=concatHex(byteArray[i],returnVal);
  }
  returnVal+="\n";
  Serial.println(returnVal);
}

void serialEvent() {
  while (Serial.available()) {
    // get the new byte:
    char inChar = (char)Serial.read();
    if(inChar == 0x11){
       char inChar = (char)Serial.read();
    } else if (inChar == 0x03) {
      seralReadEnded = true;
    } else {
     inputString+=inChar; 
    }
  }
 }

And I made the following Node.js software in order to read and send data over the USB to arduino:

const SerialPort = require('serialport');
const Readline = require('parser-readline');
const Ready = require('parser-ready');
const Regex = require('parser-regex');
const crypto = require('crypto')

const valueIntoHex=function(value){
  switch(value){
    case 0x11:
      return "0x11";
    case 0x03:
      return "0x03";
    case 0x02:
      return "0x02";
    default:
      return value;
  }
}

/**
* Escaping NonPrintable Ascii Characters (bytes) of a buffer data
* @param <Buffer> data The data to read
*/
const formatDataForSerialSend=function(data){
  //I allocate double the length because I assume worst case scenario that all bytes to be the ones I need to escape
  const returnBuffer =  Buffer.alloc(2*data.length);
  indexToWrite=0;
  //I use indexToWrite index because during escapes the posision that I write the byte is different from the ne I read.
  for(let i=0;i<data.length;i++){
    const byte=data.readUInt8(i);

    if(byte == 0x11 || byte == 0x03 || byte == 0x02){
      console.log("Escaped Ascii: "+valueIntoHex(byte));
      indexToWrite=returnBuffer.writeUInt8(0x11,indexToWrite);
    }
    indexToWrite=returnBuffer.writeUInt8(byte,indexToWrite);
  }
  returnBuffer.writeUInt8(0x03,indexToWrite);
  //Because I do not want uneeded size of data in my buffer
  const lastIndex=returnBuffer.lastIndexOf(0x03);
  return returnBuffer.slice(0,lastIndex+1);//I want size and not the last index. (Size=lastIndex+1)
}


/**
* Application Specific send logic
* @param <Buffer> data Data te send
*/
const sendData=function(data){
  console.log("Escaping dirty Ascii");
  data=formatDataForSerialSend(data);
  console.log("SEND DATA: "+data.toString('hex'));
  port.write(data,'binary',(err)=>{
    if(err){
      console.err("Could not send data",err);
    }
  });
}

const portName = '/dev/ttyACM0';

const port = new SerialPort(portName, {baudRate: 19200});

//Look over: https://regex101.com/r/QcBTTj/1
const hexRegex=/((\\|0)(x|X)[0-9A-Fa-f]{2})+/g;
const parser = port.pipe(new Ready({ delimiter: 'BEGIN:' })).pipe(new Readline({ delimiter: '\r\n' }));


console.log("################# SEND ####################");
sendData(Buffer.from('hello','ascii'));
console.log("############# END OF SEND #################");



parser.on('data', function(input) {
    console.log("################# RECIEVE ##################");
    console.log("Message: "+input);

    const input2=input.replace(/^BEGIN:|(\r|\n|(\\|0)(x|X))/g,"");
    console.log("Sanitized Input: ",input2);

    try{
        const data=Buffer.from(input2,'hex')
        console.log("REPRINT HEX FOR VERIFICATION: "+data.toString('hex'));
        console.log("############ END RECIEVE ##############");
    } catch(e) {
        console.error(e);
    }
});

But for some reason I managed to make it send data but the arduino does not send them back. So I want to debug it, but I do not know how I can do that I mean over arduino code how I will launch a debugger in order to do that?

How I can at the same time look and analyze the USB traffic between an application and Arduino in order to determine the logic bug?

Also I could not distinguish why arduino does not echo back some data, if I use this code for this sketch, the node.js application reads the data fine:

/*
 * Copyright 2018 Dimitrios Desyllas
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, 
 * including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 
 * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

void setup() {
  Serial.begin(9600);
  while (!Serial);
}

void loop() {
  byte byteArray[]={1,10,255,3,4,2,1,55};
  for(int i=0;i<sizeof(byteArray);i++){
    Serial.print("\\x");
    if(byteArray[i]<16){Serial.print(0);}
    Serial.print(byteArray[i],HEX);
  }
  Serial.println("\n");
  delay(3000);
}

Edit1

So far I modified the sketch into this:

String inputString = "";         // a String to hold incoming data
bool seralReadEnded=false;


void setup() {
  Serial.begin(19200);
  while (!Serial);
}

void loop() {
  if(seralReadEnded){
    sendDataOverSerial((byte [])inputString.c_str(),inputString.length());
    inputString="";
    seralReadEnded=false;
  }
}

String concatHex(byte value,String string){
  string+="\\x";
  if(value<16){
   string+="0";
  }
  string+=String(value,HEX);
  return string;
}

void sendDataOverSerial(byte byteArray[],size_t arrayLength){
  String returnVal="BEGIN:";
  for(int i=0;i<arrayLength;i++){
    returnVal=concatHex(byteArray[i],returnVal);
  }
  returnVal+="\n";
  Serial.println(returnVal);
}

void serialEvent() {
  while (Serial.available()) {
    // get the new byte:
    char inChar = (char)Serial.read();
    if(inChar == 0x11){
       char inChar = (char)Serial.read();
    } else if (inChar == 0x03) {
      seralReadEnded = true;
    } else {
     inputString+=inChar; 
    }
  }
 }

And I tested it by running over my GNU/Linux terminal like that:

screen /dev/ttyACM0 19200

I typed some stuff and by pressing CTRL+C keyboard combination I managed to get them back (Somehow gives a character 0x03 that is the ETX ascii character that in my case indicated the data stream).

But How I get some random binary escape them with 0x11 charactes and send into arduino via bash, as fas ad I know is that on GNU/Linux systems I can use the /dev/random for generating data?

Answer 1

There are too many String copies in your code. This is asking for trouble, as it can lead to memory fragmentation and, ultimately, to memory exhaustion. That's why I recommended in a comment to buffer the data in plain byte arrays.

However, it appears your Node.js script is going to do the buffering anyway (that's what the Readline parser does). Thus there is no point in buffering also on the Arduino, which has one million (or more) times less memory than the computer running the Node.js code. If the Arduino just outputs the bytes as it gets them, the program becomes very simple:

void setup()
{
    Serial.begin(19200);
    Serial.print("BEGIN:");
}

// Send a byte over the serial port, formatted in hex.
static void send_hex(uint8_t value)
{
    Serial.print("\\x");
    if (value < 16)
        Serial.print('0');
    Serial.print(value, HEX);
}

void loop()
{
    static bool escaped;  // is the next character escaped?
    if (Serial.available()) {
        char c = Serial.read();
        if (c == '\x10' && !escaped) {  // DLE (data link escape)
            escaped = true;
        } else if (c == '\x03' && !escaped) {  // ETX (end of text)
            Serial.print("\r\nBEGIN:");
        } else {
            send_hex(c);
            escaped = false;
        }
    }
}

Note that, upon receiving an “end of text”, the program terminates the current line (by sending "\r\n") and immediately sends the preamble for the next one ("BEGIN:"). On the receiving side, the readline parser will take care of delivering only complete lines.

If, for whatever reason, you do need to buffer inside the Arduino, then remove the Serial.print("BEGIN:"); line from setup() and change the loop as follows:

void loop()
{
    static char buffer[1024];
    static size_t pos;    // current position in buffer
    static bool escaped;  // is the next character escaped?
    if (Serial.available()) {
        char c = Serial.read();
        if (c == '\x10' && !escaped) {  // DLE (data link escape)
            escaped = true;
        } else if (c == '\x03' && !escaped) {  // ETX (end of text)
            Serial.print("BEGIN:");
            for (size_t i = 0; i < pos; i++)
                send_hex(buffer[i]);
            Serial.println();
            pos = 0;
        } else {
            if (pos < sizeof buffer)
                buffer[pos++] = c;
            escaped = false;
        }
    }
}

Note that the buffer is sized at half the RAM of your Uno. This is safe, as it is a very simple program that uses very little dynamic memory (stack only, no heap) and has no risk of fragmentation. Note also that your original program stores both the raw data and the formatted hex (which is 4 times larger) in memory at the same time, which means your buffering would be limited to 1/5 of the available RAM.