Debugging different serial receive / send behavior between Uno & Mega

Question

I have a simple text parser (as a component of a larger project) coded up and seemingly working correctly. It parses a key-value delimited command like the following: <key1=value1;key2=value2;key3=value3>.

At present, the functionality is very silly just to test it out: when the Arduino finds a key named "text", it composes a response of a similar form, but just with one key "text_return" and the same value, and sends it back via serial. That is, when the command <key1=value1;text=123;key3=value3> is sent, I expect a response from the Arduino of <text_return=123>.

I then am connecting the board to my laptop via USB, and testing it out, both with the Serial Monitor in Arduino's IDE, and with a simple program coded in python.

So here's the confusing part:

1. Serial Monitor with Uno: works
2. Python with Uno: works
3. Serial Monitor with Mega: works
4. Python with Mega: does NOT work

Specifically, by "does not work", I mean that the Mega seems to receive the serial input from the program, but does not issue a response.

Here's the terminal output from test #2 as displayed by the python program, showing the command-response I'd expect (I parsed the received command into a dict):

SENT: <text=1953>
SENT: <text=1954>
SENT: <text=1954>
SENT: <text=1954>
SENT: <text=1954>
SENT: <text=1954>
SENT: <text=1954>
SENT: <text=1954>
RECD: {'text_return': 1954}
SENT: <text=1954>
RECD: {'text_return': 1954}
SENT: <text=1954>
RECD: {'text_return': 1954}
SENT: <text=1954>
RECD: {'text_return': 1954}

And here's the terminal output from the non-working test#4 - i.e.: no response.

SENT: <text=2001>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2002>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>
SENT: <text=2003>

In all cases, I'm using 9600 Baud; the only change between the two tests is that I select the Uno vs. Mega / Mega 2560 board in the Arduino IDE.

The Mega board is an Elegoo Mega2560 R3. The code may be extraneous, but to help:

Opening the port to the arduino from python

def FindArduino(baud=9600, timeout=0):
  initial_time = time.time()
  arduino_found = False
  attempted = False
  while not attempted or time.time() - initial_time < timeout and not arduino_found:
    attempted = True
    ports = serial.tools.list_ports.comports(include_links=False)
    for port in ports:
      manufacturer = port.manufacturer
      if manufacturer and 'arduino' in manufacturer.lower():
        arduino_port = port
        arduino_found = True

  if arduino_found:
    try:
      arduino = serial.Serial(arduino_port.device, baud, timeout=0)
      arduino.reset_input_buffer()
      arduino.reset_output_buffer()
    except serial.SerialException:
      arduino_found = False

  if arduino_found:
    return arduino
  else:
    return None

Sending the serial in python, to Arduino

  cmd = '<text=%d>' % (time.time() - 1586218411)
  print('SENT: %s' %cmd)
  cmd = bytes(cmd, 'ascii')
  arduino.write(cmd)
  time.sleep(.1)

Receiving the serial in python, from Arduino

  cmds = []

  if (arduino.in_waiting>0):
    buffer += arduino.read(arduino.in_waiting).decode('ascii')
    while COMMAND_END_CHAR in buffer:
      end_char_pos = buffer.find(COMMAND_END_CHAR)
      potential_command = buffer[:end_char_pos]
      if COMMAND_START_CHAR in potential_command:
        cmds.append(potential_command[potential_command.find(COMMAND_START_CHAR)+1:])
      buffer = buffer[end_char_pos+1:]

  return (buffer, cmds)

And the full code on the Arduino itself

// command indicators
char START_MARKER = '<';
char END_MARKER = '>';

const int MAX_KEY_VALUE_PAIRS = 3;  // maximum number of key-value pairs in message
const int MAX_ELEMENT_CHARS = 30;  // the maximum number of characters (+1 for terminator) in a key or a value

// message format: <key1=value1;key2=value2;key3=value3>
const int MAX_MESSAGE_CHARS = (MAX_KEY_VALUE_PAIRS * (MAX_ELEMENT_CHARS + 1)) * 2 + (MAX_KEY_VALUE_PAIRS - 1) + 2;  // maximum message size

char received_chars[MAX_MESSAGE_CHARS];
bool new_data = false;
char written_chars[MAX_MESSAGE_CHARS];

char *text =  {'\0'};


void ParseData(char *str) {
  // This picks off the ;-delimited key-value pairs and assigns them to a multi-dim array
  char * pch;
  int pairs_count = 0;
  char config[MAX_KEY_VALUE_PAIRS][2][MAX_ELEMENT_CHARS];

  pch = strtok(str, "=");

  while (pch != NULL)
  {
    strcpy(config[pairs_count][0], pch);
    pch = strtok(NULL, ";");
    if (pch == NULL) break;

    strcpy(config[pairs_count][1], pch);
    pairs_count++;
    pch = strtok(NULL, "=");
    if (pch == NULL) break;
  }

  for(int i=0;i<pairs_count;i++) {
    if (strcmp(config[i][0], "text")==0) 
      strcpy(text, config[i][1]);
    WriteSerial();
  }
}

void ReadSerial(){
  // After calling ReceiveText to empty the buffer, if a complete command has been found,
  // parse that command.
  new_data = ReceiveText();
  if (new_data == true) {
    char temp_chars[MAX_MESSAGE_CHARS];  // temporary array for use when parsing
    strcpy(temp_chars, received_chars);
    received_chars[0] = '\0';
    ParseData(temp_chars);
  }
}

void WriteSerial(){
  sprintf(written_chars, "<text_return=%s>\n", text);
  Serial.write(written_chars);
  written_chars[0] = '\0';
}

boolean ReceiveText() {
  // This dumps the characters on the buffer so far received_chars, searching for an END_MARKER
  // along the way; if it finds one, it goes back to find a START_MARKER; if that is also found,
  // the string within is the ;-delimited set of key value pairs
  static boolean recv_in_progress = false;
  static byte ndx = 0;
  char rc;

  boolean new_data = false;
  while (Serial.available() > 0 && new_data == false) {
    rc = Serial.read();

    if (recv_in_progress == true) {
      if (rc != END_MARKER) {
        received_chars[ndx] = rc;
        ndx++;
        if (ndx >= MAX_MESSAGE_CHARS) {
          ndx = MAX_MESSAGE_CHARS - 1;
        }
      } else {
        received_chars[ndx] = '\0'; // terminate the string
        recv_in_progress = false;
        ndx = 0;
        new_data = true;
      }
    } else if (rc == START_MARKER) {
        recv_in_progress = true;
    }
  }
  return new_data;
}

void setup() {  
  Serial.begin(9600);
  text = (char*)malloc(25);

  received_chars[0] = '\0';

}

void loop() {
  ReadSerial();
}

UPDATE

Using the suggested libraries, I updated the code. The comms in both directions for simple test data are working, though I can't seem to convert the packet received by the Arduino into a string correctly. Here is the output I see from my python terminal; I'd expect to receive back the first character of what was sent (i.e.: the "0", then "1"), rather than just the initialized value of "S".

SENT: 0.20
RCVD: S
SENT: 0.41
RCVD: S
SENT: 0.61
RCVD: S
SENT: 0.82
RCVD: S
SENT: 1.02
RCVD: S
SENT: 1.23
RCVD: S
SENT: 1.43
RCVD: S
SENT: 1.64
RCVD: S

Python code

import time
from pySerialTransfer import pySerialTransfer as txfer

if __name__ == '__main__':
  try:
    link = txfer.SerialTransfer('/dev/cu.usbmodem14201')

    link.open()
    time.sleep(2) # allow some time for the Arduino to completely reset
    base = time.time()

    while True:
      time.sleep(0.2)
      s = '%.2f' % (time.time() - base)
      l = len(s)
      for i in range(l):
        link.txBuff[i] = s[i]

      link.send(l)

      while not link.available():
        if link.status < 0:
          print('ERROR: {}'.format(link.status))


      response = ''
      for index in range(link.bytesRead):
        response += chr(link.rxBuff[index])

      print('SENT: %s' % s)
      print('RCVD: %s' % response)

  except KeyboardInterrupt:
    link.close()

Arduino code

#include "SerialTransfer.h"

char str[100];

SerialTransfer myTransfer;
int LED = 13;

void blinkLED(int n)
{
  for (int i = 0; i < n; i++) {
    digitalWrite(LED, HIGH);
    delay(150);
    digitalWrite(LED, LOW);
    delay(150);
  }
} 


void setup()
{
  pinMode(LED, OUTPUT);
  digitalWrite(LED, LOW);
  str[0] = 'S';
  str[1] = '\n';
  Serial.begin(115200);
  myTransfer.begin(Serial);
}

void loop()
{
  blinkLED(2);
  delay(500);

  // send bytes
  myTransfer.txBuff[0] = str[0];
  myTransfer.sendData(1);
  if(myTransfer.available())
  {
    blinkLED(60);

    // receive bytes
    byte bytes_to_read = myTransfer.bytesRead;
    for(byte i = 0; i < bytes_to_read; i++)
      strncpy(str + i, myTransfer.rxBuff[i], 1);

  }
  else if(myTransfer.status < 0)
  {
    Serial.print("ERROR: ");

    if(myTransfer.status == -1)
      Serial.println(F("CRC_ERROR"));
    else if(myTransfer.status == -2)
      Serial.println(F("PAYLOAD_ERROR"));
    else if(myTransfer.status == -3)
      Serial.println(F("STOP_BYTE_ERROR"));
  }
}

Thank you!

Answer 1

It would be easier if you used Arduino <--> Python compatible libraries to ensure the communication between the two is robust and reliable. An example of such libraries would be pySerialTransfer and SerialTransfer.h.

pySerialTransfer is pip-installable and cross-platform compatible. SerialTransfer.h runs on the Arduino platform and can be installed through the Arduino IDE's Libraries Manager.

Both of these libraries have highly efficient and robust packetizing/parsing algorithms with easy to use APIs.

Example Python Script:

from time import sleep
from pySerialTransfer import pySerialTransfer as txfer

if __name__ == '__main__':
    try:
        link = txfer.SerialTransfer('COM17')

        link.open()
        sleep(2) # allow some time for the Arduino to completely reset

        while True:
            link.txBuff[0] = 'h'
            link.txBuff[1] = 'i'
            link.txBuff[2] = '\n'

            link.send(3)

            while not link.available():
                if link.status < 0:
                    print('ERROR: {}'.format(link.status))

            print('Response received:')

            response = ''
            for index in range(link.bytesRead):
                response += chr(link.rxBuff[index])

            print(response)

    except KeyboardInterrupt:
        link.close()

Example Arduino Sketch:

#include "SerialTransfer.h"

SerialTransfer myTransfer;

void setup()
{
  Serial.begin(115200);
  myTransfer.begin(Serial);
}

void loop()
{
  myTransfer.txBuff[0] = 'h';
  myTransfer.txBuff[1] = 'i';
  myTransfer.txBuff[2] = '\n';

  myTransfer.sendData(3);
  delay(100);

  if(myTransfer.available())
  {
    //do stuff with received packet data
  }
  else if(myTransfer.status < 0)
  {
    Serial.print("ERROR: ");

    if(myTransfer.status == -1)
      Serial.println(F("CRC_ERROR"));
    else if(myTransfer.status == -2)
      Serial.println(F("PAYLOAD_ERROR"));
    else if(myTransfer.status == -3)
      Serial.println(F("STOP_BYTE_ERROR"));
  }
}

Note that you can send more than just individual chars with these libraries. It is possible to transfer floats, ints, bytes, arrays, and even structs (or any combination of the like) within your program using the libraries! See the examples in SerialTransfer.h for more info

For theory behind robust serial communication, check out the tutorials Serial Input Basics and Serial Input Advanced.

EDIT:

---

Applying the above example to your case, you can use the following code:

Python:

import time
import struct
from pySerialTransfer import pySerialTransfer as txfer


def stuff_float(txfer_obj, val, start_pos=0):
    '''
    Description:
    ------------
    Insert a 32-bit floating point value into the (pySerialtxfer) TX
    buffer starting at the specified index

    :param txfer_obj: txfer - Transfer class instance to communicate over serial
    :param val:       float - value to be inserted into TX buffer
    :param start_pos: int   - index of TX buffer where the first byte of
                              the float is to be stored in

    :return start_pos: int - index of the last byte of the float in the TX
                             buffer + 1
    '''

    val_bytes = struct.pack('f', val)

    txfer_obj.txBuff[start_pos] = val_bytes[0]
    start_pos += 1
    txfer_obj.txBuff[start_pos] = val_bytes[1]
    start_pos += 1
    txfer_obj.txBuff[start_pos] = val_bytes[2]
    start_pos += 1
    txfer_obj.txBuff[start_pos] = val_bytes[3]
    start_pos += 1

    return start_pos


if __name__ == '__main__':
  try:
    link = txfer.SerialTransfer('COM17')

    link.open()
    time.sleep(2) # allow some time for the Arduino to completely reset
    base = time.time()

    while True:
      time.sleep(0.2)

      sent = time.time() - base
      stuff_float(link, sent)
      link.send(4)

      while not link.available():
        if link.status < 0:
          print('ERROR: {}'.format(link.status))

      response = ''
      for index in range(link.bytesRead):
        response += chr(link.rxBuff[index])

      print('SENT: {}'.format(sent))
      print('RCVD: {}'.format(response))
      print(' ')

  except KeyboardInterrupt:
    link.close()

Arduino:

#include "SerialTransfer.h"


SerialTransfer myTransfer;


char buff[25];
char str[] = "I got this: %s%d.%04d";
float fromPython = 0;


void setup()
{
  Serial.begin(115200);
  myTransfer.begin(Serial);
}

void loop()
{
  if(myTransfer.available())
  {
    //////////////////////////////////////////////
    // handle call from Python
    myTransfer.rxObj(fromPython, sizeof(fromPython));

    char sign = (fromPython < 0) ? '-' : ' ';
    uint16_t integer = fromPython;
    float tmpFrac = fromPython - integer;
    uint16_t decimal = trunc(tmpFrac * 10000);
    sprintf(buff, str, sign, integer, decimal);
    //////////////////////////////////////////////

    //////////////////////////////////////////////
    // send response
    myTransfer.txObj(buff, sizeof(buff));
    myTransfer.sendData(sizeof(buff));
    //////////////////////////////////////////////
  }
}

Verified Python Output:

SENT: 0.20032477378845215
RCVD: I got this: 0.2003

SENT: 0.4077117443084717
RCVD: I got this: 0.4077

SENT: 0.61326003074646
RCVD: I got this: 0.6132

Yet another edit:

---

Inspired by the answer submitted by OP, I updated the library (1.2.0) to include the member functions tx_obj() and rx_obj(). These will allow you to automatically send and receive nearly any standard Python object types (including lists and dictionaries).

Example Python Code:

import time
from pySerialTransfer import pySerialTransfer as txfer


if __name__ == '__main__':
    try:
        link = txfer.SerialTransfer('COM17')

        link.open()
        time.sleep(2) # allow some time for the Arduino to completely reset

        while True:
            send_size = 0

            ###################################################################
            # Send a list
            ###################################################################
            list_ = [1, 3]
            list_size = link.tx_obj(list_)
            send_size += list_size

            ###################################################################
            # Send a string
            ###################################################################
            str_ = 'hello'
            str_size = link.tx_obj(str_, send_size) - send_size
            send_size += str_size

            ###################################################################
            # Send a float
            ###################################################################
            float_ = 5.234
            float_size = link.tx_obj(float_, send_size) - send_size
            send_size += float_size

            ###################################################################
            # Transmit all the data to send in a single packet
            ###################################################################
            link.send(send_size)

            ###################################################################
            # Wait for a response and report any errors while receiving packets
            ###################################################################
            while not link.available():
                if link.status < 0:
                    if link.status == -1:
                        print('ERROR: CRC_ERROR')
                    elif link.status == -2:
                        print('ERROR: PAYLOAD_ERROR')
                    elif link.status == -3:
                        print('ERROR: STOP_BYTE_ERROR')

            ###################################################################
            # Parse response list
            ###################################################################
            rec_list_  = link.rx_obj(obj_type=type(list_),
                                     obj_byte_size=list_size,
                                     list_format='i')

            ###################################################################
            # Parse response string
            ###################################################################
            rec_str_   = link.rx_obj(obj_type=type(str_),
                                     obj_byte_size=str_size,
                                     start_pos=list_size)

            ###################################################################
            # Parse response float
            ###################################################################
            rec_float_ = link.rx_obj(obj_type=type(float_),
                                     obj_byte_size=float_size,
                                     start_pos=(list_size + str_size))

            ###################################################################
            # Display the received data
            ###################################################################
            print('SENT: {} {} {}'.format(list_, str_, float_))
            print('RCVD: {} {} {}'.format(rec_list_, rec_str_, rec_float_))
            print(' ')

    except KeyboardInterrupt:
        link.close()

    except:
        import traceback
        traceback.print_exc()

        link.close()

Example Arduino Code:

#include "SerialTransfer.h"


SerialTransfer myTransfer;


void setup()
{
  Serial.begin(115200);
  myTransfer.begin(Serial);
}


void loop()
{
  if(myTransfer.available())
  {
    // send all received data back to Python
    for(uint16_t i=0; i < myTransfer.bytesRead; i++)
      myTransfer.txBuff[i] = myTransfer.rxBuff[i];

    myTransfer.sendData(myTransfer.bytesRead);
  }
}

Answer 2

Building on P_B's answer to pack additional data types, I extended the python as follows. Of course, the receiving Arduino code needs to receive into the correct data type as well.

def StuffInt(txfer_obj, int_to_send, start_pos=0):
  """Insert integer into pySerialtxfer TX buffer starting at the specified index."""
  return StuffObject(txfer_obj, int_to_send, 'i', 4, start_pos=0)


def StuffFloat(txfer_obj, float_to_send, start_pos=0):
  """Insert integer into pySerialtxfer TX buffer starting at the specified index."""
  return StuffObject(txfer_obj, float_to_send, 'f', 4, start_pos=0)


def StuffStr(txfer_obj, string_to_send, max_length=None, start_pos=0):
  """Insert string into pySerialtxfer TX buffer starting at the specified index.

  Args:
    txfer_obj: see StuffObject
    string_to_send: see StuffObject
    max_length: if provided, the string is truncated to the requested size; otherwise
      defaults to the length of the string
    object_byte_size: integer number of bytes of the object to pack
    start_pos: see StuffObject
  Returns:
    start_pos for next object
  """
  if max_length is None:
    max_length = len(string_to_send)
  format_string = '%ds' % max_length
  truncated_string = string_to_send[:max_length]
  truncated_string_b = bytes(truncated_string, 'ascii')
  print (truncated_string_b)
  return StuffObject(txfer_obj, truncated_string_b, format_string, max_length, start_pos=0)


def StuffObject(txfer_obj, val, format_string, object_byte_size, start_pos=0):
  """Insert an object into pySerialtxfer TX buffer starting at the specified index.

  Args:
    txfer_obj: txfer - Transfer class instance to communicate over serial
    val: value to be inserted into TX buffer
    format_string: string used with struct.pack to pack the val
    object_byte_size: integer number of bytes of the object to pack
    start_pos: index of the last byte of the float in the TX buffer + 1

  Returns:
    start_pos for next object
  """
  val_bytes = struct.pack(format_string, val)
  for index in range(object_byte_size):
    txfer_obj.txBuff[index + start_pos] = val_bytes[index]
  return object_byte_size + start_pos