Demo 34: firmware update OTA for ESP32 using HTTP and sdcard

Tech It Yourself 6:30 PM
1. Introduction
In Demo 16, we knew how to update firmware OTA for a batch of Arduino ESP32. Actually, the method behind that tutorial is UDP protocol. In this tutorial I will show you another way to update firmware for ESP OTA. That is using HTTP and sdcard. It means we will use Http to download the firmware from Http Apache server to sdcard and then flashing downloaded firmware from sdcard to ESP.
Note: In case you do not want to store firmware file in sdcard, I will make another demo without using sdcard Demo 35.
We will re-use these demos for this tutorial:
+ Demo 14: How to use MQTT and Arduino ESP32 to build a simple Smart home system
+ Demo 7: How to use Arduino ESP32 to store data to sdcard
+ Demo 1: Blinky - a Hello World on Arduino ESP32
+ How to turn the Orange Pi/Raspberry Pi into an IoT node (this tutorial will show you how to install Apache2 HTTP server to hold firmware .bin file. Following the steps in 2.4 of the tutorial and copy the .bin file to "/var/www/html/phphello". So the url to download the firmware file (led.bin) is: http://192.168.1.107/phphello/led.bin)

Note: the firmware file "led.bin" is exported from Arduino IDE. It is a simple LED blink application that  blink the LED on pin GPIO4. In order to export the .bin file from Arduino IDE Menu, we choose Sketch -> Export compiled Binary. After finishing we choose Sketch -> Show Sketch Folder. You will se the .bin file there, rename it as led.bin

 The model of this demo is below:

Figure: Model of demo
ESP will receive MQTT command (including url to download .bin firmware file and md5 value of that file to verify whether downloading and flashing firmware is correct or not) from Python application "fotacontrol.py". After downloading and flashing process was finished the ESP send "ok" to Python application to notify the updating process was finished.

2. Hardware
Refer Demo 7 to connect ESP32 pins with sdcard module. And Demo 1 to connect ESP to LED.
3. Software
I made the library in github. Just download, install it and run example esp32httpota2. Change the Wifi ssid, password and IP of MQTT server according to yours. The code will be explained below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#include <WiFi.h>
#include <HttpOTA.h>
#include <mySD.h>
#include <PubSubClient.h>

typedef enum {
  Runnning_e = 0x01,
  Fota_e  
}SysState;

const char* ssid = "dd-wrt";
const char* password = "0000000000";
const char* mqtt_server = "192.168.1.107";

char url[100];
char md5[50];
File file;
WiFiClient espClient;
PubSubClient client(espClient);
SysState state = Runnning_e;

/* topics */
#define OTA_TOPIC    "smarthome/room1/ota"

void saveData(uint8_t *buffer, int bytes){
  file.write(buffer, bytes);
}
int readData(uint8_t *buffer, int bytes){
  return file.read(buffer, bytes);
}
void progress(DlState state, int percent){
  Serial.printf("state = %d - percent = %d\n", state, percent);
}

void receivedCallback(char* topic, byte* payload, unsigned int length) {
  
  if(strncmp(OTA_TOPIC, topic, strlen(OTA_TOPIC)) == 0){
    memset(url, 0, 100);
    memset(md5, 0, 50);
    char *tmp = strstr((char *)payload, "url:");
    char *tmp1 = strstr((char *)payload, ",");
    memcpy(url, tmp+strlen("url:"), tmp1-(tmp+strlen("url:")));
    
    char *tmp2 = strstr((char *)payload, "md5:");
    memcpy(md5, tmp2+strlen("md5:"), length-(tmp2+strlen("md5:")-(char *)&payload[0]));

    Serial.printf("started fota url: %s\n", url);
    Serial.printf("started fota md5: %s\n", md5);
    state = Fota_e;
  }
}
void mqttconnect() {
  /* Loop until reconnected */
  while (!client.connected()) {
    Serial.print("MQTT connecting ...");
    /* client ID */
    String clientId = "ESP32Client";
    /* connect now */
    if (client.connect(clientId.c_str())) {
      Serial.println("connected");
      /* subscribe topic */
      client.subscribe(OTA_TOPIC);
    } else {
      Serial.print("failed, status code =");
      Serial.print(client.state());
      Serial.println("try again in 5 seconds");
      /* Wait 5 seconds before retrying */
      delay(5000);
    }
  }
}

void error(char *message){
  printf("%s\n", message);
}

void startDl(void){
  //write bin file to sdcard
  SD.remove("fw.bin");
  file = SD.open("fw.bin", FILE_WRITE);
}
void endDl(void){
  file.close();
}
void startFl(void){
  //write bin file to sdcard
  file = SD.open("fw.bin", FILE_READ);
}
void endFl(void){
  file.close();
}
void setup() {
  // put your setup code here, to run once:
  // put your setup code here, to run once:
  Serial.begin(115200);
  Serial.print("Connecting to "); 
  Serial.print(ssid);
  WiFi.begin(ssid, password);
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  } 
  Serial.println("");
  
  Serial.print("WiFi connected, IP address: "); 
  Serial.println(WiFi.localIP());
  
  if (!SD.begin(32, 14, 12, 27)) {
    Serial.println("initialization failed!");
    return;
  }
  Serial.println("initialization done.");

  /* configure the MQTT server with IPaddress and port */
  client.setServer(mqtt_server, 1883);
  /* this receivedCallback function will be invoked 
  when client received subscribed topic */
  client.setCallback(receivedCallback);
}
 
void loop() {
  switch(state)
  {
    case Runnning_e:
      /* if client was disconnected then try to reconnect again */
      if (!client.connected()) {
        mqttconnect();
      }
      /* this function will listen for incomming 
      subscribed topic-process-invoke receivedCallback */
      client.loop();
    break;
    case Fota_e:
      DlInfo info;
      info.url = url;
      info.md5 = md5;
      info.startDownloadCallback =  startDl;
      info.endDownloadCallback =    endDl;
      info.startFlashingCallback =  startFl;
      info.endFlashingCallback =    endFl;
      
      info.saveDataCallback = saveData;
      info.readDataCallback = readData;
      info.progressCallback  = progress;
      info.errorCallback     = error;
      httpOTA.start(info);
    
      client.publish(OTA_TOPIC, "ok");
    break;
    default:
    break;
  }
}
In order to start fota updating process, using the Python code below and type "1" to publish FOTA command to ESP. The ESP will receive  the command, parse it and starting FOTA process. Change the url and md5 value according to yours.
The library is simple. It only has 1 interface that is start with input is DlInfo structure. When calling this function, it will block the loop() function. We must initialize the structure before using it. The structure includes:
- url: is the http url of the firmware file.
- md5: is the md5 checksum of the firmware file.
- startDownloadCallback: is the function that will be invoked before starting downloading. I used it to open firmware file in sdcard for writing downloaded data.
- endDownloadCallback: is the function that will be invoked after downloading was finished. I used it to close firmware file in sdcard after finishing writing downloaded data.
- startFlashingCallback: is the function that will be invoked before starting flashing downloaded firmware. I used it to open firmware file for flashing.
- endFlashingcallback: is the function that will be invoked after flashing was finished. I used it to close firmware file after finishing flashing.
- saveDataCallback: is the function that will be invoked to save downloading data to sdcard. I made it in generic way so that the library can be applied for SPIFFS.
- readDataCallback: is the function that will be invoked to read data from sdcard for flashing. I made it in generic way so that the library can be applied for SPIFFS.
- progressCallback: is the function that will be invoked to show the progress of downloading and flashing process.
- errorCallback: is the function that will be invoked to show the error of downloading and flashing process.
The functions: saveData, readData, progress, error, startDl, endDl, startFl, endFl will be invoked by the library in the update process. They are assigned to the members of DlInfo structure.
The function receiveCallback will be invoked whenever ESP received the MQTT command. It will check if the topic is OTA_TOPIC then parsing the payload to get url and md5 value of firmware file. After that, it change the state of system to Fota_e to start updating process.
After finishing the whole process, the code "client.publish(OTA_TOPIC, "ok");" will publish message "ok" back to Python application. 
When the system is in Running_e, it just listen the MQTT command.
The code "SD.begin(32, 14, 12, 27)" is ti initialize SD card before operating on it.
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from random import randint
import thread
import sys
try:
    import paho.mqtt.client as mqtt
except ImportError:
    import os
    import inspect
    cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile( inspect.currentframe() ))[0],"../src")))
    if cmd_subfolder not in sys.path:
        sys.path.insert(0, cmd_subfolder)
    import paho.mqtt.client as mqtt


server = "192.168.1.107";
topic = "smarthome/room1/ota"


def on_connect(mqttc, obj, flags, rc):
    print("rc: "+str(rc))

def on_message(mqttc, obj, msg):
    print(msg.topic+" "+str(msg.qos)+" "+str(msg.payload))

def on_publish(mqttc, obj, mid):
    print("mid: "+str(mid))

def on_subscribe(mqttc, obj, mid, granted_qos):
    print("Subscribed: "+str(mid)+" "+str(granted_qos))

def on_log(mqttc, obj, level, string):
    print(string)

mqttc = mqtt.Client()
mqttc.on_message = on_message
mqttc.on_connect = on_connect
mqttc.on_publish = on_publish
mqttc.on_subscribe = on_subscribe
mqttc.connect(server, 1883, 60)
mqttc.subscribe(topic, 0)


def fotaControl( threadName, delay):
    while True:
        val = raw_input('Enter 1 to update firmware OTA ')
        if(val == "1"):
            mqttc.publish(topic, "url:http://192.168.1.107/phphello/led.bin,md5:6bd07139c21f572370242905c4465056")

try:
    thread.start_new_thread( fotaControl, ("Fota Control", 0, ) )
except:
    print "Error: unable to start thread"

mqttc.loop_forever()
The Python application is quite simple, It has 2 thread: listening MQTT thread and fota control thread to send FOTA command to ESP (pressing "1" to start FOTA).
Before using paho MQTT we need to initialize some callback functions: on_message, on_connect, on_publish, on_subscribe.
4. Result



Post a Comment

5 Comments

Shaurya said…
Thanks for the wonderful work
Can you help me to flash esp32 from sd card having already a binary file

Thanks in advance.
February 15, 2018 at 12:52 AM
Tech It Yourself said…
Hi,

You can reuse the code from library. In detail, using this file:
https://github.com/nhatuan84/esp32-http-ota/blob/master/HttpOTA/src/HttpOTA.cpp
and remove all the code from line 25 to line 71. After that set download variable and total variable to the size of binary file. and fill info.md5 with the md5 value of binary file. It should work.

Regards
February 17, 2018 at 4:41 AM
Unknown said…
Hello, can you help me please:

http.begin (info.url);
int httpCode = http.GET ();

In the file "HttpOTA.cpp", I get the error "-1"

I have already printed the url that arrives to the file with:

Serial.println (String (info.url));

and it is correct, if I put it on my phone or on a PC I download the binary file without problems.

I am using an ESP32 connected with a SIM900, and I get the instruction correctly, as well as the url and the MD5 by MQTT, however when executing "http.begin" it is when take the fail.

Any ideas?

Thanks and regards, Gonzalo Fuentes
July 15, 2018 at 1:27 PM
ITduet said…
Hi! If I understood correctly we don't need more separate ESP flash to half for OCTA? We can use available almost .all flash space for OCTA! If ESP has 4Mb flash then update size is almost all flash size? If YES then this is super great! Thanks!
October 27, 2018 at 4:34 AM
Post a Comment