Demo 39: ESP32/8266 multipart upload a file and download a file via HTTP

1. Introduction
In previous demos, I showed you how to use MQTT/MQTTS and how to update firmware OTA (TCP/UDP and HTTP). In this demo, I will show you another interesting topic. That is how to download and multi-part upload a file via HTTP.
Figure: multipart upload and download a file via http
Based on this demo, you can build an application that can download the new firmware or the configuration file to memory card that is attached to client board for using later. Or you can upload the log file that records occurred events in run time to a server.
This is the requirement of this demo: a ESP32 will download a file from internet and after finish downloading, upload it to local apache server. The downloaded file will be stored in sdcard.
In order to make it easy, I created a library here in github. You just install and use it.
Note: this library can be apply for ESP8266.
2. Software
The library is simple to use. It has 2 APIs: upload() and download() with parameters. The parameters are:
- url of uploading and downloading file.
- progress displaying callback function.
- response processing callback function.
- reading and writing data to storage (sdcard, SPIFFS, ...) callback functions.
You should change them accordingly to your application.
Note: The url parser of this library is quite simple and it will be updated later.
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
#include "UDHttp.h"
#include "mySD.h"


const char* ssid     = "dd-wrt";
const char* password = "0000000000";

File root;
//these callbacks will be invoked to read and write data to sdcard
//and process response
//and showing progress 
int responsef(uint8_t *buffer, int len){
  Serial.printf("%s\n", buffer);
  return 0;
}
//read data callback
int rdataf(uint8_t *buffer, int len){
  //read file to upload
  if (root.available()) {
    return root.read(buffer, len);
  }
  return 0;
}
//write data callback
int wdataf(uint8_t *buffer, int len){
  //write downloaded data to file
  return root.write(buffer, len);
}

void progressf(int percent){
  Serial.printf("%d\n", percent);
}

void setup() {
  // put your setup code here, to run once:
  Serial.begin(115200);

  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
      delay(500);
      Serial.print(".");
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
    
  Serial.print("Initializing SD card...");
  if (!SD.begin(32, 14, 12, 27)) {
    Serial.println("initialization failed!");
    return;
  }
  Serial.println("initialization done.");
  SD.remove("test.pdf");
  {
    UDHttp udh;
    //open file on sdcard to write
    root = SD.open("test.pdf", FILE_WRITE);
    if (!root) {
       Serial.println("can not open file!");
       return;
    }
    //download the file from url
    udh.download("http://www.smart-words.org/linking-words/linking-words.pdf", wdataf, progressf);
    root.close();
    Serial.printf("done downloading\n");
  }
  {
    UDHttp udh;
    //open file on sdcard to read
    root = SD.open("test.pdf");
    if (!root) {
       Serial.println("can not open file!");
       return;
    }
    //upload downloaded file to local server
    udh.upload("http://192.168.1.107:80/upload/upload.php", "test.pdf", root.size(), rdataf, progressf, responsef);
    root.close();
    Serial.printf("done uploading\n");
  }
}

void loop() {
  // put your main code here, to run repeatedly:

}
And the php script was used for this demo:
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
<?php
$target_path = "uploads/";

$target_path = $target_path . basename( $_FILES['data']['name']); 

if(move_uploaded_file($_FILES['data']['tmp_name'], $target_path)) {
    echo "The file ".  basename( $_FILES['data']['name']). 
    " has been uploaded";
} else{
    echo "There was an error uploading the file, please try again!";
}
?>
Note: The variable $_FILES['data']['name'] has fixed 'data' field name (similar to html form ""). The downloaded file will be stored in sub-folder 'uploads' which is relative to folder where the php script located. Reference this post to install local apache server. 
3. Hardware
For harware connection please refer here.

4. Result
Figure: start downloading with progress displaying

Post a Comment

12 Comments

Akshay said…
hi thanks for the post
I have updated the code to upload a file from SPIFFS but there is one problem

while the code is running on my Sparkfun esp 32 things and server is offline it shows error and when I start the server it shows upload successfully on esp 32 serial monitor.
but in both cased nothing is shown on the server.

what do you think is the cause of this problem

my code

#include "UDHttp.h"
//#include "mySD.h"
#include "FS.h"
#include "SPIFFS.h"

const char* ssid = "bhai ka internet";
const char* password = "bhaibhai";

File root;
//these callbacks will be invoked to read and write data to sdcard
//and process response
//and showing progress
int responsef(uint8_t *buffer, int len){
Serial.printf("%s\n", buffer);
return 0;
}

int rdataf(uint8_t *buffer, int len){
//read file to upload
if (root.available()) {
return root.read(buffer, len);
}
return 0;
}

int wdataf(uint8_t *buffer, int len){
//write downloaded data to file
return root.write(buffer, len);
}

void progressf(int percent){
Serial.printf("%d\n", percent);
}

void setup() {
// put your setup code here, to run once:
Serial.begin(115200);

WiFi.begin(ssid, password);

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());

Serial.print("Initializing SPIFFS ...");
if (!SPIFFS.begin()) {
Serial.println("SPIFFS Mount Failed");
return;
}
Serial.println("initialization done.");
//SD.remove("test.pdf");
{
UDHttp udh;
//open file on sdcard to write
root = SPIFFS.open("/test.pdf", FILE_WRITE);
if (!root) {
Serial.println("can not open file!");
return;
}
//download the file from url
udh.download("http://www.smart-words.org/linking-words/linking-words.pdf", wdataf, progressf);
root.close();
Serial.printf("done downloading\n");
}

}

void loop() {
{
UDHttp udh;
//open file on sdcard to read
root = SPIFFS.open("/test.pdf");
if (!root) {
Serial.println("can not open file!");
return;
}
//upload downloaded file to local server
udh.upload("http://192.168.31.187:3000/coins","/test.pdf", root.size(), rdataf, progressf, responsef);
root.close();
Serial.printf("done uploading\n");
}
// put your main code here, to run repeatedly:
delay (3000);
}


tiklez107 said…
Hello!
How can I continuously upload the files in my sd card/spiffs to the external server as they come in? Also, how can I upload multiple files with the same extension (i.e. jpg) at once?
Thanks!
Daemach said…
Thank you very much for providing this library. It is exactly what I needed to finish an IoT project. Unfortunately, the upload portion which is the part I need, is not working on an ESP-32 with 1.02-rc2 Arduino code. Here is the result of the code (integer divide by zero error) - Can you help, please? You obviously very good at this and I would appreciate the help immensely:

....
WiFi connected
IP address:
192.168.2.104
Initializing SD card...initialization done.
100
done downloading
Guru Meditation Error: Core 1 panic'ed (IntegerDivideByZero). Exception was unhandled.
Core 1 register dump:
PC : 0x400d1f87 PS : 0x00060630 A0 : 0x800d1c2c A1 : 0x3ffb1c30
A2 : 0x3ffb1f77 A3 : 0x00000000 A4 : 0x00000000 A5 : 0x00000000
A6 : 0x400d1a4c A7 : 0x3f4011bc A8 : 0x800d1f78 A9 : 0x400d1a28
A10 : 0x00000000 A11 : 0x3ffb1c58 A12 : 0x3ffb1d48 A13 : 0x00000000
A14 : 0x00000063 A15 : 0xff000000 SAR : 0x00000009 EXCCAUSE: 0x00000006
EXCVADDR: 0x00000000 LBEG : 0x4000c2e0 LEND : 0x4000c2f6 LCOUNT : 0xffffffff

Backtrace: 0x400d1f87:0x3ffb1c30 0x400d1c29:0x3ffb1f60 0x400d69b3:0x3ffb1fb0 0x400889bd:0x3ffb1fd0

Rebooting...

And here is the code I used - I had to modify it slightly from the example file to get it to run.


#include "UDHttp.h"
#include "FS.h"
#include "SD.h"
#include "SPI.h"

const char* ssid = "mySSID";
const char* password = "myPW";

File root;
//these callbacks will be invoked to read and write data to sdcard
//and process response
//and showing progress
int responsef(uint8_t *buffer, int len){
Serial.printf("%s\n", buffer);
return 0;
}

int rdataf(uint8_t *buffer, int len){
//read file to upload
if (root.available()) {
return root.read(buffer, len);
}
return 0;
}

int wdataf(uint8_t *buffer, int len){
//write downloaded data to file
return root.write(buffer, len);
}

void progressf(int percent){
Serial.printf("%d\n", percent);
}

void setup() {
// put your setup code here, to run once:
Serial.begin(115200);

WiFi.begin(ssid, password);

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());

Serial.print("Initializing SD card...");
if (!SD.begin(33)) {
Serial.println("initialization failed!");
return;
}
Serial.println("initialization done.");
SD.remove("test.pdf");
{
UDHttp udh;
//open file on sdcard to write
root = SD.open("/test.pdf", FILE_WRITE);
if (!root) {
Serial.println("can not open file!");
return;
}
//download the file from url
udh.download("http://www.smart-words.org/linking-words/linking-words.pdf", wdataf, progressf);
root.close();
Serial.printf("done downloading\n");
}
{
UDHttp udh;
//open file on sdcard to read
root = SD.open("/test.pdf");
if (!root) {
Serial.println("can not open file!");
return;
}
//upload downloaded file to local server
udh.upload("http://d.thcguard.com:80/dbugUpload.cfm", "test.pdf", root.size(), rdataf, progressf, responsef);
root.close();
Serial.printf("done uploading\n");
}
}
Donato said…
Hi
I'm trying to make a simple project that downloads a file and save it into my sd.
I'm using your code, but into sd is written only a file to 1k. probably the method udh.download("http://www.smart-words.org/linking-words/linking-words.pdf", wdataf, progressf);
not working. how I can test it?
thank you Donato
Arief said…
how if i want get file from https, github maybe ?
Ram Prasanth said…
Hi
I'm trying to save files to SD card not in a local server.
The Error is:

OUTPUT :

WiFi connected
IP address:
192.168.43.24
Initializing SPIFFS ...initialization done.
url is wrong

CODE :

#include "UDHttp.h"
#include "FS.h"
#include "SPIFFS.h"

const char* ssid = "CnatioN";
const char* password = "987654321";

File root;
//these callbacks will be invoked to read and write data to sdcard
//and process response
//and showing progress
int responsef(uint8_t *buffer, int len){
Serial.printf("%s\n", buffer);
return 0;
}

int rdataf(uint8_t *buffer, int len){
//read file to upload
if (root.available()) {
return root.read(buffer, len);
}
return 0;
}

int wdataf(uint8_t *buffer, int len){
//write downloaded data to file
return root.write(buffer, len);
}

void progressf(int percent){
Serial.printf("%d\n", percent);
}

void setup() {
// put your setup code here, to run once:
Serial.begin(115200);

WiFi.begin(ssid, password);

while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}

Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());

Serial.print("Initializing SPIFFS ...");
if (!SPIFFS.begin()) {
Serial.println("SPIFFS Mount Failed");
return;
}
Serial.println("initialization done.");
SPIFFS.remove("linking-words.pdf");
{
UDHttp udh;
//open file on sdcard to write
root = SPIFFS.open("/linking-words.pdf", FILE_WRITE);
if (!root) {
Serial.println("can not open file!");
return;
}
//download the file from url
udh.download("http://www.smart-words.org/linking-words/linking-words.pdf", wdataf, progressf);
root.close();
Serial.printf("done downloading\n");
}

}

void loop() {

}

done downloading
Ram Prasanth said…
Can some one tell me instead of uploading to local server i need to save my url file directly to sd card
Sharon Stone said…
PDF is gaining much attention and popularity these days, because they are portable to upload and download. Download PDF Medical pdf are good for professionals who want to keep informed of latest development in medicines and about the reasons and remedies of diseases.
Anonymous said…
Hi, I have tested the code. Thank you for sharing!


My question is, is there a chance of improve the upload transfer speeed?
I have tested this speed changing the CHUNK_SIZE from 100 up to 256.
using 512 do not work.

EPIC.MP3
8,223,529 bytes
Time uploaded: 121,341 milliseconds.
2.01 minutes
chunksize = 100


EPIC.MP3
8,223,529 bytes
Time uploaded: 105,257 milliseconds.
1.45 minutes
chunksize = 200


EPIC.MP3
8,223,529 bytes
Time uploaded: 98,888 milliseconds.
1.38 minutes
chunksize = 256


PD: For the windows side server I used this one (portable) to run in windows to receive the uploads:
https://usbwebserver.yura.mk.ua/

just unzip, run in windows.
inside \root create \upload folder and paste the upload.php there.
then create another folder inside \upload called \uploads
it will end like this: \root\upload\uploads
Unknown said…
thanks for sharing. if i want use https protocol, what should i adjust?
Bob Lynas said…
HI, Great series by the way !
I would like to see a slightly modified version of this, where my ESP32 has served a webpage and from the webpage I can select a local file and upload that file to SPIFFs in the ESP32. I am using Ethernet rather than Wifi. Regards Bob