From 07b1ea3a5ce1988d4df5d7ae157fff27def395f3 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 28 Nov 2019 06:42:30 +0100 Subject: [PATCH 01/24] Added Updater to automatically perform OTA updates without user interaction. --- DEPLOY.md | 9 +++ data/_version.txt | 1 + include/config.sample.h | 12 +++- include/controller.h | 1 + include/updater.h | 8 +++ src/controller.cpp | 9 +++ src/main.cpp | 26 +++++++- src/updater.cpp | 127 ++++++++++++++++++++++++++++++++++++++++ update.manifest | 3 + 9 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 DEPLOY.md create mode 100644 data/_version.txt create mode 100644 include/updater.h create mode 100644 src/updater.cpp create mode 100644 update.manifest diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..75dab17 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,9 @@ +# Deploying a new version + +* Update the OTA_VERSION in `include/config.h`, `include/config.sample.h`, + `data/_version.txt`, and `update.manifest`. +* Commit the changes. +* Tag the commit with the new version string. +* Push everything. +* Create theimages using `pio run` and `pio run -t buildfs`. +* Upload `update.manifest`, `firmware.bin` and `spiffs.bin`. diff --git a/data/_version.txt b/data/_version.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/data/_version.txt @@ -0,0 +1 @@ +1 diff --git a/include/config.sample.h b/include/config.sample.h index d13fff6..05fe964 100644 --- a/include/config.sample.h +++ b/include/config.sample.h @@ -1,13 +1,21 @@ #pragma once #include +// This is a simple number indicating the version for the HTTP Updater. +#define OTA_VERSION 1 +// Comment out to prevent automatic updates. +#define OTA_UPDATE_URL "https://files.schle.nz/esmp3/update.manifest" +#define OTA_CHECK_INTERVAL 12*60*60*1000 // 12 hours + #define SHOW_DEBUG //#define SHOW_TRACE #define FTP_DEBUG #define DELAY_AFTER_DEBUG_AND_TRACE 0 -#define WIFI_SSID "---CHANGEME---" -#define WIFI_PASS "---CHANGEME---" +// Here you can define WiFi data to use. But actually, the better way to do +// this is by using /_wifi.txt on the sd card. +//#define WIFI_SSID "---CHANGEME---" +//#define WIFI_PASS "---CHANGEME---" #define VS1053_SLEEP_DELAY 5000 #define POSITION_SEND_INTERVAL 5000 diff --git a/include/controller.h b/include/controller.h index 00ce87e..980281b 100644 --- a/include/controller.h +++ b/include/controller.h @@ -32,6 +32,7 @@ private: unsigned long _last_rfid_scan_at = 0; unsigned long _last_position_info_at = 0; + unsigned long _last_update_check_at = 0; String _serial_buffer = String(); String _cmd_queue = ""; void _execute_command_ls(String path); diff --git a/include/updater.h b/include/updater.h new file mode 100644 index 0000000..b781fc2 --- /dev/null +++ b/include/updater.h @@ -0,0 +1,8 @@ +#pragma once + +class Updater { +public: + static void run(); + static bool do_update(int cmd, String url); + static void update_spiffs(); +}; diff --git a/src/controller.cpp b/src/controller.cpp index 1326620..9994cd2 100644 --- a/src/controller.cpp +++ b/src/controller.cpp @@ -3,6 +3,7 @@ #include "config.h" #include "playlist.h" #include "http_server.h" +#include "updater.h" #include Controller::Controller(Player* p, PlaylistManager* playlist_manager) { @@ -50,6 +51,14 @@ void Controller::loop() { process_message(_cmd_queue); _cmd_queue = ""; } + + #ifdef OTA_UPDATE_URL + if (!player->is_playing() && _last_update_check_atselect_sd(false); DEBUG("Starting SPIFFS...\n"); + uint16_t spiffs_version = 0; SPIFFS.begin(true); + if (SPIFFS.exists("/_version.txt")) { + File f = SPIFFS.open("/_version.txt", "r"); + spiffs_version = f.readString().toInt(); + f.close(); + DEBUG("SPIFFS filesystem version is %d.\n", spiffs_version); + } else { + DEBUG("No SPIFFS filesystem version found - setting spiffs_version to 0.\n"); + } DEBUG("Initializing PlaylistManager...\n"); pm = new PlaylistManager(); @@ -93,9 +103,15 @@ void setup() { f.close(); } SPIMaster::select_sd(false); + + if (!connected) { + #if defined(WIFI_SSID) and defined(WIFI_PASS) DEBUG("Trying hardcoded WiFi data...\n"); connected = connect_to_wifi(WIFI_SSID, WIFI_PASS); + #else + DEBUG("No hardcoded WiFi data set.\n"); + #endif } if (!connected) { INFO("No WiFi connection!\n"); @@ -118,6 +134,12 @@ void setup() { } else { INFO("Could not fetch current time via NTP.\n"); } + + #ifdef OTA_UPDATE_URL + if (spiffs_version < OTA_VERSION) { + Updater::run(); + } + #endif INFO("Initialization completed.\n"); } diff --git a/src/updater.cpp b/src/updater.cpp new file mode 100644 index 0000000..9d956c3 --- /dev/null +++ b/src/updater.cpp @@ -0,0 +1,127 @@ +#include +#include +#include "config.h" +#include "updater.h" +#include "http_client_wrapper.h" + +void Updater::run() { + DEBUG("Updater is running...\n"); + HTTPClientWrapper* http = new HTTPClientWrapper(); + DEBUG("Requesting update info...\n"); + bool result = http->get(OTA_UPDATE_URL); + if (!result) { + ERROR("Updater failed requesting %s.\n", OTA_UPDATE_URL); + return; + } + String line = http->readUntil("\n"); + if (!line.startsWith("VERSION=")) { + ERROR("Expected first line to be VERSION.\n"); + return; + } + uint16_t version = line.substring(8).toInt(); + if (version==0) { + ERROR("Could not parse version number.\n"); + return; + } + DEBUG("Found version %d. My version is %d.\n", version, OTA_VERSION); + if (version <= OTA_VERSION) { + return; + } + line = http->readUntil("\n"); + if (!line.startsWith("IMAGE_PATH=")) { + ERROR("Expected second line to be IMAGE_PATH.\n"); + return; + } + String image_path = line.substring(11); + image_path.trim(); + line = http->readUntil("\n"); + if (!line.startsWith("SPIFFS_PATH=")) { + ERROR("Expected third line to be SPIFFS_PATH.\n"); + return; + } + String spiffs_path = line.substring(12); + spiffs_path.trim(); + + http->close(); + delete http; + + result = do_update(U_FLASH, image_path); + if (result) { + do_update(U_SPIFFS, spiffs_path); + } + DEBUG("Done. Rebooting...\n"); + ESP.restart(); +} + +bool Updater::do_update(int command, String url) { + HTTPClientWrapper* http = new HTTPClientWrapper(); + bool result = http->get(url); + if (!result) { + ERROR("Updater failed requesting %s.\n", url.c_str()); + return false; + } + + result = Update.begin(http->getSize(), command); + if (!result) { + ERROR("Update could not be started.\n"); + return false; + } + uint8_t buf[512]; + uint16_t len; + while((len = http->read(buf, 512))) { + Update.write(buf, 512); + } + + http->close(); + delete http; + + result = Update.end(); + if (!result) { + ERROR("Writing the update failed somewhere. Aborted.\n"); + return false; + } + return true; +} + +void Updater::update_spiffs() { + HTTPClientWrapper* http = new HTTPClientWrapper(); + DEBUG("Requesting update info...\n"); + bool result = http->get(OTA_UPDATE_URL); + if (!result) { + ERROR("Updater failed requesting %s.\n", OTA_UPDATE_URL); + return; + } + String line = http->readUntil("\n"); + if (!line.startsWith("VERSION=")) { + ERROR("Expected first line to be VERSION.\n"); + return; + } + uint16_t version = line.substring(8).toInt(); + if (version==0) { + ERROR("Could not parse version number.\n"); + return; + } else if (version > OTA_VERSION) { + DEBUG("Found newer version %d. My version is %d. Starting full update!\n", version, OTA_VERSION); + run(); + } else { + DEBUG("Loading SPIFFS image.\n"); + line = http->readUntil("\n"); + if (!line.startsWith("IMAGE_PATH=")) { + ERROR("Expected second line to be IMAGE_PATH.\n"); + return; + } + line = http->readUntil("\n"); + if (!line.startsWith("SPIFFS_PATH=")) { + ERROR("Expected third line to be SPIFFS_PATH.\n"); + return; + } + String spiffs_path = line.substring(12); + spiffs_path.trim(); + do_update(U_SPIFFS, spiffs_path); + DEBUG("Done. Rebooting...\n"); + delay(500); + ESP.restart(); + } + http->close(); + delete http; +} diff --git a/update.manifest b/update.manifest new file mode 100644 index 0000000..69f7d45 --- /dev/null +++ b/update.manifest @@ -0,0 +1,3 @@ +VERSION=1 +IMAGE_PATH=https://files.schle.nz/esmp3/firmware.bin +SPIFFS_PATH=https://files.schle.nz/esmp3/spiffs.bin From b805d1b18399895c7142acc6f0c87ceac1878001 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 28 Nov 2019 06:48:16 +0100 Subject: [PATCH 02/24] Updater: Better flow for performing updates; added MD5 validation. --- include/updater.h | 8 ++-- src/main.cpp | 4 +- src/updater.cpp | 113 ++++++++++++++++++++-------------------------- update.manifest | 2 + 4 files changed, 58 insertions(+), 69 deletions(-) diff --git a/include/updater.h b/include/updater.h index b781fc2..3aa76b7 100644 --- a/include/updater.h +++ b/include/updater.h @@ -1,8 +1,10 @@ #pragma once +#include "http_client_wrapper.h" + class Updater { public: - static void run(); - static bool do_update(int cmd, String url); - static void update_spiffs(); + static void run(uint16_t spiffs_version = 0); + static bool do_update(int cmd, String url, String expected_md5); + static bool read_line(String* dst, HTTPClientWrapper* http, String expected_key); }; diff --git a/src/main.cpp b/src/main.cpp index e32d8d7..7b5804d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -136,9 +136,7 @@ void setup() { } #ifdef OTA_UPDATE_URL - if (spiffs_version < OTA_VERSION) { - Updater::run(); - } + Updater::run(spiffs_version); #endif INFO("Initialization completed.\n"); diff --git a/src/updater.cpp b/src/updater.cpp index 9d956c3..e0a9351 100644 --- a/src/updater.cpp +++ b/src/updater.cpp @@ -4,7 +4,8 @@ #include "updater.h" #include "http_client_wrapper.h" -void Updater::run() { +void Updater::run(uint16_t spiffs_version) { + bool update_image = true; DEBUG("Updater is running...\n"); HTTPClientWrapper* http = new HTTPClientWrapper(); DEBUG("Requesting update info...\n"); @@ -13,47 +14,74 @@ void Updater::run() { ERROR("Updater failed requesting %s.\n", OTA_UPDATE_URL); return; } - String line = http->readUntil("\n"); - if (!line.startsWith("VERSION=")) { - ERROR("Expected first line to be VERSION.\n"); + + String line_str = ""; + if (!read_line(&line_str, http, "VERSION")) { return; } - uint16_t version = line.substring(8).toInt(); + uint16_t version = line_str.toInt(); if (version==0) { ERROR("Could not parse version number.\n"); return; } DEBUG("Found version %d. My version is %d.\n", version, OTA_VERSION); if (version <= OTA_VERSION) { + if (spiffs_version>0 && spiffs_version < version) { + update_image = false; + DEBUG("SPIFFS needs an update. Continuing.\n"); + } else { + return; + } + } + + String image_path = ""; + if (!read_line(&image_path, http, "IMAGE_PATH")) { return; } - line = http->readUntil("\n"); - if (!line.startsWith("IMAGE_PATH=")) { - ERROR("Expected second line to be IMAGE_PATH.\n"); + + String image_md5 = ""; + if (!read_line(&image_md5, http, "IMAGE_MD5")) { return; } - String image_path = line.substring(11); - image_path.trim(); - line = http->readUntil("\n"); - if (!line.startsWith("SPIFFS_PATH=")) { - ERROR("Expected third line to be SPIFFS_PATH.\n"); + + String spiffs_path = ""; + if (!read_line(&spiffs_path, http, "SPIFFS_PATH")) { + return; + } + + String spiffs_md5 = ""; + if (!read_line(&spiffs_md5, http, "SPIFFS_MD5")) { return; } - String spiffs_path = line.substring(12); - spiffs_path.trim(); http->close(); delete http; - result = do_update(U_FLASH, image_path); + result = true; + if (update_image) { + result = do_update(U_FLASH, image_path, image_md5); + } if (result) { - do_update(U_SPIFFS, spiffs_path); + do_update(U_SPIFFS, spiffs_path, spiffs_md5); } DEBUG("Done. Rebooting...\n"); ESP.restart(); } -bool Updater::do_update(int command, String url) { +bool Updater::read_line(String* dst, HTTPClientWrapper* http, String expected_key) { + expected_key += "="; + String line = http->readUntil("\n"); + if (!line.startsWith(expected_key)) { + ERROR("Expected line start with '%s', but it started with '%s'.\n", expected_key.c_str(), line.c_str()); + return false; + } + line = line.substring(expected_key.length()); + line.trim(); + dst->concat(line); + return true; +} + +bool Updater::do_update(int command, String url, String expected_md5) { HTTPClientWrapper* http = new HTTPClientWrapper(); bool result = http->get(url); if (!result) { @@ -66,10 +94,11 @@ bool Updater::do_update(int command, String url) { ERROR("Update could not be started.\n"); return false; } + Update.setMD5(expected_md5.c_str()); uint8_t buf[512]; uint16_t len; while((len = http->read(buf, 512))) { - Update.write(buf, 512); + Update.write(buf, len); } http->close(); @@ -77,51 +106,9 @@ bool Updater::do_update(int command, String url) { result = Update.end(); if (!result) { - ERROR("Writing the update failed somewhere. Aborted.\n"); + const char* error = Update.errorString(); + ERROR("Writing the update failed. The error was: %s\n", error); return false; } return true; } - -void Updater::update_spiffs() { - HTTPClientWrapper* http = new HTTPClientWrapper(); - DEBUG("Requesting update info...\n"); - bool result = http->get(OTA_UPDATE_URL); - if (!result) { - ERROR("Updater failed requesting %s.\n", OTA_UPDATE_URL); - return; - } - String line = http->readUntil("\n"); - if (!line.startsWith("VERSION=")) { - ERROR("Expected first line to be VERSION.\n"); - return; - } - uint16_t version = line.substring(8).toInt(); - if (version==0) { - ERROR("Could not parse version number.\n"); - return; - } else if (version > OTA_VERSION) { - DEBUG("Found newer version %d. My version is %d. Starting full update!\n", version, OTA_VERSION); - run(); - } else { - DEBUG("Loading SPIFFS image.\n"); - line = http->readUntil("\n"); - if (!line.startsWith("IMAGE_PATH=")) { - ERROR("Expected second line to be IMAGE_PATH.\n"); - return; - } - line = http->readUntil("\n"); - if (!line.startsWith("SPIFFS_PATH=")) { - ERROR("Expected third line to be SPIFFS_PATH.\n"); - return; - } - String spiffs_path = line.substring(12); - spiffs_path.trim(); - do_update(U_SPIFFS, spiffs_path); - DEBUG("Done. Rebooting...\n"); - delay(500); - ESP.restart(); - } - http->close(); - delete http; -} diff --git a/update.manifest b/update.manifest index 69f7d45..7f0bdd5 100644 --- a/update.manifest +++ b/update.manifest @@ -1,3 +1,5 @@ VERSION=1 IMAGE_PATH=https://files.schle.nz/esmp3/firmware.bin +IMAGE_MD5=ead8d94ae8a3c9f46e7ee65e2270fd69 SPIFFS_PATH=https://files.schle.nz/esmp3/spiffs.bin +SPIFFS_MD5=9f3902e3312863da8326e7fbe9485c9e From bbf77c6b1eb3fb126004c6a2e865a285d3225239 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Thu, 28 Nov 2019 19:26:35 +0100 Subject: [PATCH 03/24] Added deploy script. --- deploy.sh | 37 +++++++++++++++++++++++++++++++++++++ platformio.ini | 21 +++++++++++++++++---- update.manifest | 5 ----- 3 files changed, 54 insertions(+), 9 deletions(-) create mode 100755 deploy.sh delete mode 100644 update.manifest diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..e13af7c --- /dev/null +++ b/deploy.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -x +set -e + +if ! git diff-index --quiet HEAD ; then + echo "Git isn't clean. Cant deploy." + exit 1 +fi + +read -p "Version to generate: " VERSION + +OTA_VERSION=`grep "VERSION=" bin/update.manifest | cut -d"=" -f2` +OTA_VERSION=$(( "$OTA_VERSION" + 1 )) +echo "$OTA_VERSION" > data/_version.txt + +sed -i.bak "s/#define OTA_VERSION .*/#define OTA_VERSION $OTA_VERSION/" include/config.h include/config.sample.h +rm include/config.h.bak include/config.sample.h.bak + +PLATFORMIO_BUILD_FLAGS='-DVERSION=\"$VERSION\"' pio run -e deploy -t buildprog || exit 1 +pio run -e deploy -t buildfs || exit 1 + +cp .pio/build/deploy/firmware.bin bin/firmware.bin || exit 1 +cp .pio/build/deploy/spiffs.bin bin/spiffs.bin || exit 1 + +sed -i.bak "s/VERSION=.*/VERSION=$OTA_VERSION/" bin/update.manifest +MD5=`md5sum --binary bin/firmware.bin | cut -d" " -f1` +sed -i.bak "s/IMAGE_MD5=.*/IMAGE_MD5=$MD5/" bin/update.manifest +MD5=`md5sum --binary bin/spiffs.bin | cut -d" " -f1` +sed -i.bak "s/SPIFFS_MD5=.*/SPIFFS_MD5=$MD5/" bin/update.manifest +rm bin/update.manifest.bak + +git diff +exit 99 + +git add bin/firmware.bin bin/spiffs.bin bin/update.manifest +git commit -m "Deploying version $VERSION." +git tag -a -m "Deploying version $VERSION" $VERSION diff --git a/platformio.ini b/platformio.ini index a3cfd34..f9800a5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -8,16 +8,29 @@ ; Please visit documentation for the other options and examples ; https://docs.platformio.org/page/projectconf.html +[platformio] +default_envs = esp32 + +[extra] +lib_deps = + 63 ; MFRC522 + https://github.com/me-no-dev/ESPAsyncWebServer.git + ArduinoJSON + 6691 ; TinyXML + [env:esp32] platform = espressif32 board = esp-wrover-kit framework = arduino upload_speed = 512000 build_flags=!./build_version.sh -lib_deps = MFRC522 - https://github.com/me-no-dev/ESPAsyncWebServer.git - ArduinoJSON - 6691 ; TinyXML +lib_deps = ${extra.lib_deps} upload_port = /dev/cu.SLAB_USBtoUART monitor_speed = 74480 ;monitor_port = /dev/cu.wchusbserial1420 + +[env:deploy] +platform = espressif32 +board = esp-wrover-kit +framework = arduino +lib_deps = ${extra.lib_deps} diff --git a/update.manifest b/update.manifest deleted file mode 100644 index 7f0bdd5..0000000 --- a/update.manifest +++ /dev/null @@ -1,5 +0,0 @@ -VERSION=1 -IMAGE_PATH=https://files.schle.nz/esmp3/firmware.bin -IMAGE_MD5=ead8d94ae8a3c9f46e7ee65e2270fd69 -SPIFFS_PATH=https://files.schle.nz/esmp3/spiffs.bin -SPIFFS_MD5=9f3902e3312863da8326e7fbe9485c9e From 7a20cf4b04a81b715401eddd618e816a5810a96c Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 29 Nov 2019 05:29:01 +0100 Subject: [PATCH 04/24] Better deploy script. --- DEPLOY.md | 8 +------- deploy.sh | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/DEPLOY.md b/DEPLOY.md index 75dab17..6120bc0 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,9 +1,3 @@ # Deploying a new version -* Update the OTA_VERSION in `include/config.h`, `include/config.sample.h`, - `data/_version.txt`, and `update.manifest`. -* Commit the changes. -* Tag the commit with the new version string. -* Push everything. -* Create theimages using `pio run` and `pio run -t buildfs`. -* Upload `update.manifest`, `firmware.bin` and `spiffs.bin`. +* Use `deploy.sh`. diff --git a/deploy.sh b/deploy.sh index e13af7c..3ec8bca 100755 --- a/deploy.sh +++ b/deploy.sh @@ -7,6 +7,15 @@ if ! git diff-index --quiet HEAD ; then exit 1 fi +branch_name=$(git symbolic-ref -q HEAD) +branch_name=${branch_name##refs/heads/} +branch_name=${branch_name:-HEAD} + +if [ "$branch_name" != "master" ]; then + echo "We are not on master branch. Can't deploy." + exit 1 +fi + read -p "Version to generate: " VERSION OTA_VERSION=`grep "VERSION=" bin/update.manifest | cut -d"=" -f2` @@ -29,9 +38,13 @@ MD5=`md5sum --binary bin/spiffs.bin | cut -d" " -f1` sed -i.bak "s/SPIFFS_MD5=.*/SPIFFS_MD5=$MD5/" bin/update.manifest rm bin/update.manifest.bak +echo; echo; echo; echo; echo +echo "Please check the git diff, if everything looks okay:" git diff -exit 99 + +read -p "Press ENTER to continue, Ctrl-C to abort. " foo git add bin/firmware.bin bin/spiffs.bin bin/update.manifest git commit -m "Deploying version $VERSION." git tag -a -m "Deploying version $VERSION" $VERSION +git push --follow-tags From 6445dc0fb8e6668c51d59d923a0125c3909ac8f3 Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 29 Nov 2019 05:32:22 +0100 Subject: [PATCH 05/24] Unly call Updater on startup if the SPIFFS needs an update. --- src/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 7b5804d..e95d555 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -136,7 +136,9 @@ void setup() { } #ifdef OTA_UPDATE_URL - Updater::run(spiffs_version); + if (spiffs_version < OTA_VERSION) { + Updater::run(spiffs_version); + } #endif INFO("Initialization completed.\n"); From 5fe66fdaefe866fa29d67019d9d55a0bcbb2fddb Mon Sep 17 00:00:00 2001 From: Fabian Schlenz Date: Fri, 29 Nov 2019 05:35:47 +0100 Subject: [PATCH 06/24] Added a command `update` to run the update check. --- data/index.html | 2 ++ src/controller.cpp | 9 ++++----- src/http_server.cpp | 39 +++++++++++++++++++++++++++++++-------- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/data/index.html b/data/index.html index 8b1c53f..b4decb4 100644 --- a/data/index.html +++ b/data/index.html @@ -151,6 +151,7 @@
Actions
+