4

How can I do a readable, small, smart and good URL decoder in Arduino?

Now I'm using this code:

String GetRidOfurlCharacters(String urlChars)
{

  urlChars.replace("%0D%0A", String('\n'));

  urlChars.replace("+",   " ");
  urlChars.replace("%20", " ");
  urlChars.replace("%21", "!");
  urlChars.replace("%22", String(char('\"')));
  urlChars.replace("%23", "#");
  urlChars.replace("%24", "$");
  urlChars.replace("%25", "%");
  urlChars.replace("%26", "&");
  urlChars.replace("%27", String(char(39)));
  urlChars.replace("%28", "(");
  urlChars.replace("%29", ")");
  urlChars.replace("%2A", "*");
  urlChars.replace("%2B", "+");
  urlChars.replace("%2C", ",");
  urlChars.replace("%2D", "-");
  urlChars.replace("%2E", ".");
  urlChars.replace("%2F", "/");
  urlChars.replace("%30", "0");
  urlChars.replace("%31", "1");
  urlChars.replace("%32", "2");
  urlChars.replace("%33", "3");
  urlChars.replace("%34", "4");
  urlChars.replace("%35", "5");
  urlChars.replace("%36", "6");
  urlChars.replace("%37", "7");
  urlChars.replace("%38", "8");
  urlChars.replace("%39", "9");
  urlChars.replace("%3A", ":");
  urlChars.replace("%3B", ";");
  urlChars.replace("%3C", "<");
  urlChars.replace("%3D", "=");
  urlChars.replace("%3E", ">");
  urlChars.replace("%3F", "?");
  urlChars.replace("%40", "@");
  urlChars.replace("%41", "A");
  urlChars.replace("%42", "B");
  urlChars.replace("%43", "C");
  urlChars.replace("%44", "D");
  urlChars.replace("%45", "E");
  urlChars.replace("%46", "F");
  urlChars.replace("%47", "G");
  urlChars.replace("%48", "H");
  urlChars.replace("%49", "I");
  urlChars.replace("%4A", "J");
  urlChars.replace("%4B", "K");
  urlChars.replace("%4C", "L");
  urlChars.replace("%4D", "M");
  urlChars.replace("%4E", "N");
  urlChars.replace("%4F", "O");
  urlChars.replace("%50", "P");
  urlChars.replace("%51", "Q");
  urlChars.replace("%52", "R");
  urlChars.replace("%53", "S");
  urlChars.replace("%54", "T");
  urlChars.replace("%55", "U");
  urlChars.replace("%56", "V");
  urlChars.replace("%57", "W");
  urlChars.replace("%58", "X");
  urlChars.replace("%59", "Y");
  urlChars.replace("%5A", "Z");
  urlChars.replace("%5B", "[");
  urlChars.replace("%5C", String(char(65)));
  urlChars.replace("%5D", "]");
  urlChars.replace("%5E", "^");
  urlChars.replace("%5F", "_");
  urlChars.replace("%60", "`");
  urlChars.replace("%61", "a");
  urlChars.replace("%62", "b");
  urlChars.replace("%63", "c");
  urlChars.replace("%64", "d");
  urlChars.replace("%65", "e");
  urlChars.replace("%66", "f");
  urlChars.replace("%67", "g");
  urlChars.replace("%68", "h");
  urlChars.replace("%69", "i");
  urlChars.replace("%6A", "j");
  urlChars.replace("%6B", "k");
  urlChars.replace("%6C", "l");
  urlChars.replace("%6D", "m");
  urlChars.replace("%6E", "n");
  urlChars.replace("%6F", "o");
  urlChars.replace("%70", "p");
  urlChars.replace("%71", "q");
  urlChars.replace("%72", "r");
  urlChars.replace("%73", "s");
  urlChars.replace("%74", "t");
  urlChars.replace("%75", "u");
  urlChars.replace("%76", "v");
  urlChars.replace("%77", "w");
  urlChars.replace("%78", "x");
  urlChars.replace("%79", "y");
  urlChars.replace("%7A", "z");
  urlChars.replace("%7B", String(char(123)));
  urlChars.replace("%7C", "|");
  urlChars.replace("%7D", String(char(125)));
  urlChars.replace("%7E", "~");
  urlChars.replace("%7F", "Â");
  urlChars.replace("%80", "`");
  urlChars.replace("%81", "Â");
  urlChars.replace("%82", "â");
  urlChars.replace("%83", "Æ");
  urlChars.replace("%84", "â");
  urlChars.replace("%85", "â¦");
  urlChars.replace("%86", "â");
  urlChars.replace("%87", "â¡");
  urlChars.replace("%88", "Ë");
  urlChars.replace("%89", "â°");
  urlChars.replace("%8A", "Å");
  urlChars.replace("%8B", "â¹");
  urlChars.replace("%8C", "Å");
  urlChars.replace("%8D", "Â");
  urlChars.replace("%8E", "Ž");
  urlChars.replace("%8F", "Â");
  urlChars.replace("%90", "Â");
  urlChars.replace("%91", "â");
  urlChars.replace("%92", "â");
  urlChars.replace("%93", "â");
  urlChars.replace("%94", "â");
  urlChars.replace("%95", "â¢");
  urlChars.replace("%96", "â");
  urlChars.replace("%97", "â");
  urlChars.replace("%98", "Ë");
  urlChars.replace("%99", "â¢");
  urlChars.replace("%9A", "Å¡");
  urlChars.replace("%9B", "âº");
  urlChars.replace("%9C", "Å");
  urlChars.replace("%9D", "Â");
  urlChars.replace("%9E", "ž");
  urlChars.replace("%9F", "Ÿ");
  urlChars.replace("%A0", "Â");
  urlChars.replace("%A1", "¡");
  urlChars.replace("%A2", "¢");
  urlChars.replace("%A3", "£");
  urlChars.replace("%A4", "¤");
  urlChars.replace("%A5", "Â¥");
  urlChars.replace("%A6", "¦");
  urlChars.replace("%A7", "§");
  urlChars.replace("%A8", "¨");
  urlChars.replace("%A9", "©");
  urlChars.replace("%AA", "ª");
  urlChars.replace("%AB", "«");
  urlChars.replace("%AC", "¬");
  urlChars.replace("%AE", "®");
  urlChars.replace("%AF", "¯");
  urlChars.replace("%B0", "°");
  urlChars.replace("%B1", "±");
  urlChars.replace("%B2", "²");
  urlChars.replace("%B3", "³");
  urlChars.replace("%B4", "´");
  urlChars.replace("%B5", "µ");
  urlChars.replace("%B6", "¶");
  urlChars.replace("%B7", "·");
  urlChars.replace("%B8", "¸");
  urlChars.replace("%B9", "¹");
  urlChars.replace("%BA", "º");
  urlChars.replace("%BB", "»");
  urlChars.replace("%BC", "¼");
  urlChars.replace("%BD", "½");
  urlChars.replace("%BE", "¾");
  urlChars.replace("%BF", "¿");
  urlChars.replace("%C0", "Ã");
  urlChars.replace("%C1", "Ã");
  urlChars.replace("%C2", "Ã");
  urlChars.replace("%C3", "Ã");
  urlChars.replace("%C4", "Ã");
  urlChars.replace("%C5", "Ã");
  urlChars.replace("%C6", "Ã");
  urlChars.replace("%C7", "Ã");
  urlChars.replace("%C8", "Ã");
  urlChars.replace("%C9", "Ã");
  urlChars.replace("%CA", "Ã");
  urlChars.replace("%CB", "Ã");
  urlChars.replace("%CC", "Ã");
  urlChars.replace("%CD", "Ã");
  urlChars.replace("%CE", "Ã");
  urlChars.replace("%CF", "Ã");
  urlChars.replace("%D0", "Ã");
  urlChars.replace("%D1", "Ã");
  urlChars.replace("%D2", "Ã");
  urlChars.replace("%D3", "Ã");
  urlChars.replace("%D4", "Ã");
  urlChars.replace("%D5", "Ã");
  urlChars.replace("%D6", "Ã");
  urlChars.replace("%D7", "Ã");
  urlChars.replace("%D8", "Ã");
  urlChars.replace("%D9", "Ã");
  urlChars.replace("%DA", "Ã");
  urlChars.replace("%DB", "Ã");
  urlChars.replace("%DC", "Ã");
  urlChars.replace("%DD", "Ã");
  urlChars.replace("%DE", "Ã");
  urlChars.replace("%DF", "Ã");
  urlChars.replace("%E0", "Ã");
  urlChars.replace("%E1", "á");
  urlChars.replace("%E2", "â");
  urlChars.replace("%E3", "ã");
  urlChars.replace("%E4", "ä");
  urlChars.replace("%E5", "Ã¥");
  urlChars.replace("%E6", "æ");
  urlChars.replace("%E7", "ç");
  urlChars.replace("%E8", "è");
  urlChars.replace("%E9", "é");
  urlChars.replace("%EA", "ê");
  urlChars.replace("%EB", "ë");
  urlChars.replace("%EC", "ì");
  urlChars.replace("%ED", "í");
  urlChars.replace("%EE", "î");
  urlChars.replace("%EF", "ï");
  urlChars.replace("%F0", "ð");
  urlChars.replace("%F1", "ñ");
  urlChars.replace("%F2", "ò");
  urlChars.replace("%F3", "ó");
  urlChars.replace("%F4", "ô");
  urlChars.replace("%F5", "õ");
  urlChars.replace("%F6", "ö");
  urlChars.replace("%F7", "÷");
  urlChars.replace("%F8", "ø");
  urlChars.replace("%F9", "ù");
  urlChars.replace("%FA", "ú");
  urlChars.replace("%FB", "û");
  urlChars.replace("%FC", "ü");
  urlChars.replace("%FD", "ý");
  urlChars.replace("%FE", "þ");
  urlChars.replace("%FF", "ÿ");

  return urlChars;
}

However I want to make a loop and only replace a url escaping if it exists, not trying to replace all the (possible) occurrences.

  • very good for me . but in the line urlChars.replace("%5C", String(char(65))); I have String(char(123))); .... or "\" – user25636 Aug 10 '16 at 10:26
4

The fact that you're using a String (urgh, shudder shudder... you should read this: http://hacking.majenko.co.uk/the-evils-of-arduino-strings) actually makes it a little harder to work with. Instead you should have your String data in a character array ("C string"). Then a parser is much easier to write using a pair of character pointers.

Basically the theory is you scan from the start of the string to the end of the string using both pointers copying characters as you go, and as soon as you hit a % you read ahead 2 characters using the first pointer (the second pointer stays where it is) and you convert the two read characters as hex into the ASCII equivalent and use the second pointer to replace the %.

Then both pointers keep moving forward, but they stay out of step. As each % crops up the pointers get further and further out of step until you reach the end of the string and all the %xx have been replaced with a single character.

For instance, this code I have just knocked up does the job:

char data[50] = "http://foo/bar?x=hello%20there%3A%20majenko";

// Create two pointers that point to the start of the data
char *leader = data;
char *follower = leader;

// While we're not at the end of the string (current character not NULL)
while (*leader) {
    // Check to see if the current character is a %
    if (*leader == '%') {

        // Grab the next two characters and move leader forwards
        leader++;
        char high = *leader;
        leader++;
        char low = *leader;

        // Convert ASCII 0-9A-F to a value 0-15
        if (high > 0x39) high -= 7;
        high &= 0x0f;

        // Same again for the low byte:
        if (low > 0x39) low -= 7;
        low &= 0x0f;

        // Combine the two into a single byte and store in follower:
        *follower = (high << 4) | low;
    } else {
        // All other characters copy verbatim
        *follower = *leader;
    }

    // Move both pointers to the next character:
    leader++;
    follower++;
}
// Terminate the new string with a NULL character to trim it off
*follower = 0;

The data array now contains:

http://foo/bar?x=hello there: majenko
  • +35: Thank you very much! I'm not a C/C++ programmer, I only used Bash, Python and Gambas3 with a good level ;-). I'm beginning to contribute to this project: github.com/esp8266/Basic and I wanted to simplify the above code because it seemed ugly and badly implemented. However, would you recommend me to drop the use of String on all the project, or should I convert the input String to a char? – Helio Nov 22 '15 at 17:41
  • Strings are lazy and overly wasteful of resources. Not as critical on an esp8266 as it is on an Arduino, but still not good. I never ever use them. – Majenko Nov 22 '15 at 18:18
2

I wrote a Tiny web server for the Arduino which has a similar thing in it. You might find it useful for your project. Inside there is a function that decodes the URL codes on-the-fly (ie. in a state machine). The code is a bit different to Majenko's because he assumes that you have the string in memory. In my case I am decoding a byte at a time (so you don't have to hold a long string in memory before you decode it).

The snippet that does the decoding is this:

  // percent-encoding: possible states
  enum EncodePhaseType {
    ENCODE_NONE,
    ENCODE_GOT_PERCENT,
    ENCODE_GOT_FIRST_CHAR,
  };
  // percent-encoding: current state
  EncodePhaseType encodePhase;
  byte encodeByte;  // encoded byte being assembled (first nybble)

...

// ---------------------------------------------------------------------------
// add a character to the value buffer - percent-encoded (if wanted)
// ---------------------------------------------------------------------------
 void HTTPserver::addToValueBuffer (byte inByte, const bool percentEncoded)
  {
  if (valueBufferPos >= MAX_VALUE_LENGTH)
    {
    flags |= FLAG_VALUE_BUFFER_OVERFLOW;
    return;
    }  // end of overflow

  // look for stuff like "foo+bar" (turn the "+" into a space)
  // and also "foo%21bar" (turn %21 into one character)
  if (percentEncoded)
    {
    switch (encodePhase)
      {

      // if in "normal" mode, turn a "+" into a space, and look for "%"
      case ENCODE_NONE:
        if (inByte == '+')
          inByte = ' ';
        else if (inByte == '%')
          {
          encodePhase = ENCODE_GOT_PERCENT;
          return;  // no addition to buffer yet
          }
        break;

      // we had the "%" last time, this should be the first hex digit
      case ENCODE_GOT_PERCENT:
        if (isxdigit (inByte))
          {
          byte c = toupper (inByte) - '0';
          if (c > 9)
            c -= 7;  // Fix A-F
          encodeByte = c << 4;
          encodePhase = ENCODE_GOT_FIRST_CHAR;
          return;  // no addition to buffer yet
          }
        // not a hex digit, give up
        encodePhase = ENCODE_NONE;
        flags |= FLAG_ENCODING_ERROR;
        break;

      // this should be the second hex digit
      case ENCODE_GOT_FIRST_CHAR:
        if (isxdigit (inByte))
          {
          byte c = toupper (inByte) - '0';
          if (c > 9)
            c -= 7;  // Fix A-F
          inByte = encodeByte | c;
          }
        else
          flags |= FLAG_ENCODING_ERROR;

        // done with encoding it, or not a hex digit
        encodePhase = ENCODE_NONE;

      } // end of switch on encodePhase
    } // end of percent-encoded

  // add to value buffer, encoding has been dealt with
  valueBuffer [valueBufferPos++] = inByte;
  valueBuffer [valueBufferPos] = 0;  // trailing null-terminator
  } // end of HTTPserver::addToValueBuffer

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.