Allow ETag based caching of static web pages

Send the md5 hash of static files as the file etag, for all static
files.
Send a 304 Not Modified response for requests containing a
"If-None-Match" header with the correct etag.
The server now sends a Cache-Control header with the value "public,
no-cache" for static files and "no-store" for dynamic pages.

Other changes:
 * Fix embedded files always needing to be rebuilt
 * Merge Accept openmetrics and AcceptEncoding gzip check into
csvHeaderContains
 * Make uzlib_gzip_wrapper a bit more resilient against missing input

.gitignore

@@ -10,6 +10,9 @@
 # Python
 __pycache__
 
+# MyPy
+.mypy_cache
+
 # Credentials
 wifissid.txt
 wifipass.txt

TODO.md

@@ -12,13 +12,13 @@ Not all of these are necessarily going to be implemented at all.
  * Use an asynchronous DHT library(for example https://github.com/bertmelis/esp32DHT/)?
  * Add (stream?) compression to uzlib_gzip_wrapper
  * Change callback based uzlib_ungzip_wrapper to use C++ function objects instead of C function pointers
+ * Consider using PIO middleware or SCons compilation callback to generate compressed web files(into build dir?)
 
 ## Web Interface
  * Add error message to web interface if measurement fails
  * Add a javescript temperature and humidity graph to the web interface?
  * Remove not measured values(for example humidity for the DS18B20)
  * Add degrees fahrenheit mode to web interface(clientside setting)
- * Allow caching of static resources using a hash of their content as an ETag
  * Add theme switcher to web interface(clientside setting)
  * Web server IPv6 support
  * Add current commit to prometheus info and HTTP Server header
@@ -31,6 +31,10 @@ Not all of these are necessarily going to be implemented at all.
  * Add MQTT state json
  * Cleanup MQTT code
  * Add MQTT discovery support(https://www.home-assistant.io/docs/mqtt/discovery/)
- * Add WiFi info to prometheus metrics
  * Dynamically gzip compress metrics page?
  * Implement actual prometheus library as external project
+ * Add prometheus info metrics esptherm_build_info, esptherm_network_info, esptherm_module_info, and esptherm_sensor_info
+ * Add MQTT metrics to prometheus
+ * Add prometheus metrics for HTTP response times and sizes
+ * Add prometheus metrics for push statistics, if DSM is disabled
+ * Add measurement error metrics

lib/uzlib_gzip_wrapper/src/uzlib_gzip_wrapper.cpp

@@ -31,22 +31,31 @@ uzlib_ungzip_wrapper::uzlib_ungzip_wrapper(const uint8_t *cmp_start,
 		wsize = -15;
 	}
 
+	void *dict = NULL;
+	if (cmp_end < cmp_start + 29) {
+		log_e("Compressed buffer too small.");
+		log_i("A gzip compressed 0 byte file is 29 bytes in size.");
+		log_i("The given file was %d bytes.", cmp_end - cmp_start);
+	} else {
+		dict = malloc(pow(2, -wsize));
+
+		// Read uncompressed size from compressed file.
+		dlen = cmp_end[-1];
+		dlen = 256 * dlen + cmp_end[-2];
+		dlen = 256 * dlen + cmp_end[-3];
+		dlen = 256 * dlen + cmp_end[-4];
+
+	}
+
 	decomp = new uzlib_uncomp;
-	void *dict = malloc(pow(2, -wsize));
 	// Try anyways, since small files can be decompressed without one.
-	if (!dict) {
+	if (dict == NULL) {
 		log_e("Failed to allocate decompression dict.");
 	}
 
-	// Read uncompressed size from compressed file.
-	dlen = cmp_end[-1];
-	dlen = 256 * dlen + cmp_end[-2];
-	dlen = 256 * dlen + cmp_end[-3];
-	dlen = 256 * dlen + cmp_end[-4];
-
 	uzlib_uncompress_init(decomp, dict, pow(2, -wsize));
 	decomp->source = cmp_start;
-	decomp->source_limit = cmp_end - 4;
+	decomp->source_limit = cmp_end - 4 >= cmp_start ? cmp_end - 4 : cmp_start;
 	decomp->source_read_cb = NULL;
 	uzlib_gzip_parse_header(decomp);
 }

platformio.ini

@@ -28,7 +28,9 @@ default_envs =
 framework = arduino
 monitor_speed = 115200
 upload_speed = 921600
-extra_scripts = post:shared/compress_web.py
+extra_scripts =
+	pre:shared/compress_web.py
+	pre:shared/generate_hash_header.py
 lib_deps =
 	ArduinoOTA
 	ESPAsyncWebServer = https://github.com/me-no-dev/ESPAsyncWebServer.git
@@ -124,7 +126,7 @@ build_flags =
 [env:esp_wroom_02_ota]
 extends = env:esp_wroom_02
 upload_protocol = espota
-upload_port = esp-wifi-thermometer.local
+upload_port = esp-wifi-Thermometer.local
 extra_scripts =
     ${env.extra_scripts}
     post:shared/read_ota_pass.py

shared/compress_web.py

@@ -1,16 +1,22 @@
-#!/usr/bin/python
-
-Import ("env")
+#!/usr/bin/env python3
 
 from enum import Enum
 import os
-import os.path as path
+from os import path
+import sys
 
 from gzip_compressing_stream import GzipCompressingStream
 
-# Unquoted spaces will be removed from these.
+try:
+    Import ("env") # type: ignore[name-defined]
+except:
+    print("Failed to load platformio environment!", file=sys.stderr)
+    exit(1)
+
+# Text files to potentially remove the spaces from, and compress.
 input_text_files = [ 'src/html/index.html', 'src/html/error.html', 'src/html/main.css', 'src/html/index.js', 'src/html/manifest.json', 'images/favicon.svg' ]
 
+# Binary files to compress without modifying.
 input_binary_files = [ 'images/favicon.ico', 'images/favicon.png' ]
 
 # These files will not be gzip compressed, just copied and potentially stripped of spaces.
@@ -161,11 +167,11 @@ def compress_file(input, text):
         Whether the input file is a text file.
     """
 
+    do_gzip = not input in input_gzip_blacklist
+    input = path.join(env.subst('$PROJECT_DIR'), input)
     filename = path.basename(input)
     minify = text
 
-    do_gzip = not input in input_gzip_blacklist
-
     if env.get('BUILD_TYPE') == "debug":
         print("Debug mode detected, not minifying text files.")
         minify = False
@@ -215,12 +221,8 @@ def compress_file(input, text):
         filename = path.basename(file)
     else:
         filename = path.basename(file) + ".gz"
-    # Always build because the output will depend on whether we are building in debug mode.
-    env.AlwaysBuild(f"$BUILD_DIR/{filename}.txt.o")
     compress_file(file, True)
 
 for file in input_binary_files:
     filename = path.basename(file) + ".gz"
-    # Always build these because I'm too lazy to check whether they actually changed.
-    env.AlwaysBuild(f"$BUILD_DIR/{filename}.txt.o")
     compress_file(file, False)

shared/generate_hash_header.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3 
+
+import hashlib
+import os
+from os import path, PathLike
+import sys
+from typing import Dict, Final, List
+
+try:
+    Import ("env")  # type: ignore[name-defined]
+except:
+    print("Failed to load platformio environment!", file=sys.stderr)
+    sys.exit(1)
+
+# The file endings of files to generate hashes for.
+# Files also have to be in the directory specified in "static_dir" for a hash to be calculated.
+static_types: Final[List[str]] = ['.gz', '.png', '.ico', '.html', '.css', '.js']
+
+# The directory containing the files to calculate a hash for.
+static_dir: Final[str] = 'data'
+
+# A list of files that would be considered static, but should not have their hashes calculated.
+static_file_blacklist: Final[List[str]] = [path.join('data', 'index.html'), path.join('data', 'error.html')]
+
+# The path of the header file to generated.
+hash_header_path: Final[str] = path.join(env.subst('$PROJECT_SRC_DIR'), 'generated', 'web_file_hashes.h')  # type: ignore[name-defined]
+
+
+def hash_file(path: PathLike | str) -> str:
+    """Calculates the md5 hash of a file.
+
+    This function reads the content of a file and calculates its md5 hash.
+    Note that this function reads the entire file into memory at once, so it should not be used for large files.
+
+    @param path: The path of the file to hash.
+    @return The hex representation of the resulting hash.
+    @raise IOError: If opening or reading the file fails.
+    """
+    with open(path, 'rb') as file:
+        md5 = hashlib.md5(file.read())
+        return md5.hexdigest()
+
+
+def hash_files(paths: List[PathLike] | List[str]) -> Dict[str, str]:
+    """Calculates the hashes for a list of files.
+
+    Creates a dictionary mapping from the filename to its md5 hash.
+
+    @param paths: The paths of the files to hash.
+    @return A dictionary containing the file hashes.
+    @raise IOError: If reading one of the files fails.
+    """
+    hashes: Dict[str, str] = {}
+    for p in paths:
+        id: str = path.basename(p)
+        hashes[id] = hash_file(p)
+
+    return hashes
+
+
+def generate_hash_header(hashes: Dict[str, str]) -> None:
+    """Generates the header file containing the hashes of the static files.
+
+    Generates a header file, in src/generated, containing constant definitions with the hashes of the static files.
+    
+    @param hashes: The filenames and hashes to generate the header for.
+    """
+
+    print("Generating " + path.relpath(hash_header_path, env.subst("$PROJECT_ROOT"))) # type: ignore[name-defined]
+
+    header = open(hash_header_path, 'w')
+    header.write(
+"""/*
+ * web_file_hashes.h
+ *
+ * **Warning:** This file is automatically generated, and should not be edited manually.
+ *
+ * This file contains constant definitions for the hashes of the static files sent by the web server.
+ *
+ * This project is licensed under the MIT License.
+ * The MIT license can be found in the project root and at https://opensource.org/licenses/MIT.
+ */
+
+#ifndef SRC_GENERATED_WEB_FILE_HASHES_H_
+#define SRC_GENERATED_WEB_FILE_HASHES_H_
+
+""")
+
+    for file, hash in hashes.items():
+        header.write(
+f"""/**
+ * The md5 hash of the file "{file}".
+ */
+""")
+
+        id: str = file.upper()
+        for c in ['.', '-', '/', ' ']:
+            id = id.replace(c, '_')
+
+        header.write(f"static constexpr const char {id}_HASH[] = \"{hash}\";")
+        header.write(os.linesep + os.linesep)
+
+    header.write("#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */" + os.linesep)
+
+
+def main() -> int:
+    """The main entrypoint of this script.
+
+    The main function executing all the functionality of this script.
+    Hashes the static web files, and generates a header file containing those hashes.
+
+    @return Zero if nothing goes wrong.
+    @raise IOError: If opening or reading the file fails.
+    """
+
+    static_path_abs: Final[str] = path.abspath(static_dir)
+    static_path_blacklist_abs: Final[List[str]] = [path.abspath(path.join(env.subst('$PROJECT_DIR'), file.strip())) for file in static_file_blacklist]  # type: ignore[name-defined]
+
+    files: List[str] = env.GetProjectOption('board_build.embed_files', '').splitlines()  # type: ignore[name-defined]
+    files.extend(env.GetProjectOption('board_build.embed_txtfiles', '').splitlines())  # type: ignore[name-defined]
+    files = [path.abspath(path.join(env.subst('$PROJECT_DIR'), file.strip())) for file in files if file.strip()]  # type: ignore[name-defined]
+    files = [file for file in files if file.startswith(path.abspath(static_dir)) and path.splitext(file)[1] in static_types and file not in static_path_blacklist_abs]
+
+    hashes = hash_files(files)
+    generate_hash_header(hashes)
+
+    return 0
+
+
+if __name__ == '__main__' or __name__ == 'SCons.Script':
+    error: int = main()
+    if error != 0:
+        sys.exit(error)

shared/gzip_compressing_stream.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """A custom gzip compressing stream using a configurable window size.
 
 Also contains a simple command line interface."""
@@ -246,5 +248,5 @@ def main():
     return 0
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     sys.exit(main())

shared/read_ota_pass.py

@@ -1,4 +1,5 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
+
 Import ("env")
 
 import os

src/AsyncHeadOnlyResponse.cpp

@@ -9,12 +9,13 @@
  */
 
 #include "AsyncHeadOnlyResponse.h"
+#include "fallback_log.h"
 
 #if ENABLE_WEB_SERVER == 1
 web::AsyncHeadOnlyResponse::AsyncHeadOnlyResponse(
 		AsyncWebServerResponse *wrapped, const int status_code) :
 		AsyncBasicResponse(status_code), _wrapped(wrapped) {
-
+	log_d("Creating head only response.");
 }
 
 web::AsyncHeadOnlyResponse::~AsyncHeadOnlyResponse() {

src/generated/README.md

@@ -0,0 +1,3 @@
+# Generated Sources
+This directory contains automatically generated source files and **IS NOT INTENDED TO BE MANUALLY EDITED**.  
+Any changes to the files in this directory are likely to be overridden during the build process.

src/generated/web_file_hashes.h

@@ -0,0 +1,45 @@
+/*
+ * web_file_hashes.h
+ *
+ * **Warning:** This file is automatically generated, and should not be edited manually.
+ *
+ * This file contains constant definitions for the hashes of the static files sent by the web server.
+ *
+ * This project is licensed under the MIT License.
+ * The MIT license can be found in the project root and at https://opensource.org/licenses/MIT.
+ */
+
+#ifndef SRC_GENERATED_WEB_FILE_HASHES_H_
+#define SRC_GENERATED_WEB_FILE_HASHES_H_
+
+/**
+ * The md5 hash of the file "main.css.gz".
+ */
+static constexpr const char MAIN_CSS_GZ_HASH[] = "f144c285f1ef9fd820edeac7e3d79263";
+
+/**
+ * The md5 hash of the file "index.js.gz".
+ */
+static constexpr const char INDEX_JS_GZ_HASH[] = "b7d1cc7b510da0eb5044b08fd49f6d7a";
+
+/**
+ * The md5 hash of the file "manifest.json.gz".
+ */
+static constexpr const char MANIFEST_JSON_GZ_HASH[] = "0f2bc88509c2e5fa90582f041843c42f";
+
+/**
+ * The md5 hash of the file "favicon.ico.gz".
+ */
+static constexpr const char FAVICON_ICO_GZ_HASH[] = "31fc8a37562605f1d0b3c919dffa4e96";
+
+/**
+ * The md5 hash of the file "favicon.png.gz".
+ */
+static constexpr const char FAVICON_PNG_GZ_HASH[] = "f18756d0d8dbb95f5493d805443fa023";
+
+/**
+ * The md5 hash of the file "favicon.svg.gz".
+ */
+static constexpr const char FAVICON_SVG_GZ_HASH[] = "de3746021c5d6d082e271941e00f7dbc";
+
+#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */

src/prometheus.cpp

@@ -263,14 +263,11 @@ size_t prom::writeMetricMetadataLine(char *buffer,
 #endif /* ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 */
 
 #if ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1
-bool prom::acceptsOpenMetrics(const char *accept_str) {
-	const char *start = strstr(accept_str, "application/openmetrics-text");
-	return start != NULL;
-}
-
 web::ResponseData prom::handleMetrics(AsyncWebServerRequest *request) {
 	const bool openmetrics = request->hasHeader("Accept")
-			&& acceptsOpenMetrics(request->header("Accept").c_str());
+			&& web::csvHeaderContains(request->header("Accept").c_str(),
+					"application/openmetrics-text");
+
 	if (openmetrics) {
 		log_d("Client accepts openmetrics.");
 	} else {
@@ -284,7 +281,7 @@ web::ResponseData prom::handleMetrics(AsyncWebServerRequest *request) {
 							"application/openmetrics-text; version=1.0.0; charset=utf-8" :
 							"text/plain; version=0.0.4; charset=utf-8"),
 					metrics);
-	response->addHeader("Cache-Control", "no-cache");
+	response->addHeader("Cache-Control", web::CACHE_CONTROL_NOCACHE);
 	response->addHeader("Vary", "Accept");
 	return web::ResponseData(response, metrics.length(), 200);
 }