From 46142234bdf9e0eb07379f3b37127a74a261cf2e Mon Sep 17 00:00:00 2001 From: ToMe25 <38815969+ToMe25@users.noreply.github.com> Date: Thu, 4 Apr 2024 18:37:16 +0200 Subject: [PATCH] Improve prometheus metrics generation reliability That means it shouldn't crash with multiple request types to the same URI anymore. Additional changes: * Add esptherm_version header containing the current commit hash * Add constexpr uint to str utility * Add esptherm_build_info metric to metrics page * This metrics has the labels esptherm_commit, mcu_type, arduino_version, sdk_version, and cpp_std_version. --- .gitignore | 3 + TODO.md | 2 +- lib/utils/include/utils.h | 57 +++++++++++++- platformio.ini | 4 +- shared/generate_hash_header.py | 23 +++--- shared/generate_version_header.py | 105 +++++++++++++++++++++++++ src/prometheus.cpp | 123 ++++++++++++++++++++++++------ src/prometheus.h | 86 ++++++++++++++++++++- src/webhandler.cpp | 4 +- 9 files changed, 362 insertions(+), 45 deletions(-) create mode 100755 shared/generate_version_header.py diff --git a/.gitignore b/.gitignore index 9aefad9..b9fd38e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ wifipass.txt otapass.txt mqttuser.txt mqttpass.txt + +# This file changes every commit. +src/generated/esptherm_version.h diff --git a/TODO.md b/TODO.md index d37fb3d..0c82829 100644 --- a/TODO.md +++ b/TODO.md @@ -33,7 +33,7 @@ Not all of these are necessarily going to be implemented at all. * Add MQTT discovery support(https://www.home-assistant.io/docs/mqtt/discovery/) * Dynamically gzip compress metrics page? * Implement actual prometheus library as external project - * Add prometheus info metrics esptherm_build_info, esptherm_network_info, esptherm_module_info, and esptherm_sensor_info + * Add prometheus info metrics esptherm_network_info, esptherm_module_info, and esptherm_sensor_info * Add MQTT metrics to prometheus * Add prometheus metrics for HTTP response times and sizes * Add prometheus metrics for push statistics, if DSM is disabled diff --git a/lib/utils/include/utils.h b/lib/utils/include/utils.h index 92c10e3..9e7a880 100644 --- a/lib/utils/include/utils.h +++ b/lib/utils/include/utils.h @@ -76,9 +76,62 @@ constexpr size_t strlen(const char *str) { #endif /** - * EXPAND_MACRO is a utility macro to expand a macro to its string form. + * MACRO_TO_STRING is a utility macro to convert a macro to its string form. */ -#define EXPAND_MACRO(macro) #macro +#define MACRO_TO_STRING(macro) #macro + +/** + * EXPAND_MACRO is a utility macro to expand a macro value to its string form. + */ +#define EXPAND_MACRO(macro) MACRO_TO_STRING(macro) + +namespace utils { +/** + * A helper struct to convert a set of digits to a C string. + * + * @tparam digits The digits to convert to a string. + */ +template +struct digits_to_chars { static const char value[]; }; + +/** + * A helper struct to convert a set of digits to a C string. + * + * @tparam digits The digits to convert to a string. + */ +template +const char digits_to_chars::value[] = {('0' + digits)..., 0}; + +/** + * A helper struct separating a number into separate digits. + * + * @tparam rem The remainder that is yet to be separated. + * @tparam digits The digits to convert to a string. + */ +template +struct explode : explode {}; + +/** + * A helper struct separating a number into separate digits. + * + * @tparam digits The digits to convert to a string. + */ +template +struct explode<0, digits...> : digits_to_chars {}; + +/** + * A helper struct converting an unsigned number to a string. + * + * @tparam num The digits to convert to a string. + */ +template +struct unsigned_to_string : explode {}; +} /* namespace utils */ + +/** + * UNSIGNED_TO_STRING is a helper macro converting an unsigned number to a constexpr string. + */ +#define UNSIGNED_TO_STRING(exp) utils::unsigned_to_string::value namespace utils { /** diff --git a/platformio.ini b/platformio.ini index 91cf64d..e961e72 100644 --- a/platformio.ini +++ b/platformio.ini @@ -31,6 +31,7 @@ upload_speed = 921600 extra_scripts = pre:shared/compress_web.py pre:shared/generate_hash_header.py + pre:shared/generate_version_header.py lib_deps = ArduinoOTA ESPAsyncWebServer = https://github.com/me-no-dev/ESPAsyncWebServer.git @@ -52,6 +53,7 @@ board_build.embed_files = data/gzip/favicon.ico.gz data/gzip/favicon.png.gz data/gzip/favicon.svg.gz +; The C++ version is a default for platforms that don't specify one. build_flags = -std=c++11 -Wall @@ -126,7 +128,7 @@ build_flags = [env:esp_wroom_02_ota] extends = env:esp_wroom_02 upload_protocol = espota -upload_port = esp-wifi-Thermometer.local +upload_port = esp-wifi-thermometer.local extra_scripts = ${env.extra_scripts} post:shared/read_ota_pass.py diff --git a/shared/generate_hash_header.py b/shared/generate_hash_header.py index 3d74639..e118537 100755 --- a/shared/generate_hash_header.py +++ b/shared/generate_hash_header.py @@ -62,14 +62,15 @@ def generate_hash_header(hashes: Dict[str, str]) -> None: """Generates the header file containing the hashes of the static files. Generates a header file, in src/generated, containing constant definitions with the hashes of the static files. - + @param hashes: The filenames and hashes to generate the header for. + @raise IOError: If opening or writing the header file fails. """ print("Generating " + path.relpath(hash_header_path, env.subst("$PROJECT_ROOT"))) # type: ignore[name-defined] - header = open(hash_header_path, 'w') - header.write( + with open(hash_header_path, 'w') as header: + header.write( """/* * web_file_hashes.h * @@ -86,21 +87,21 @@ def generate_hash_header(hashes: Dict[str, str]) -> None: """) - for file, hash in hashes.items(): - header.write( + for file, hash in hashes.items(): + header.write( f"""/** * The md5 hash of the file "{file}". */ """) - id: str = file.upper() - for c in ['.', '-', '/', ' ']: - id = id.replace(c, '_') + id: str = file.upper() + for c in ['.', '-', '/', ' ']: + id = id.replace(c, '_') - header.write(f"static constexpr const char {id}_HASH[] = \"{hash}\";") - header.write(os.linesep + os.linesep) + header.write(f"static constexpr const char {id}_HASH[] = \"{hash}\";") + header.write(os.linesep + os.linesep) - header.write("#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */" + os.linesep) + header.write("#endif /* SRC_GENERATED_WEB_FILE_HASHES_H_ */" + os.linesep) def main() -> int: diff --git a/shared/generate_version_header.py b/shared/generate_version_header.py new file mode 100755 index 0000000..68c3a32 --- /dev/null +++ b/shared/generate_version_header.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +import os +from os import path +import subprocess +import sys +from typing import Final +from subprocess import CalledProcessError + +try: + Import ("env") # type: ignore[name-defined] +except: + print("Failed to load platformio environment!", file=sys.stderr) + sys.exit(1) + +# The path of the header file to generated. +version_header_path: Final[str] = path.join(env.subst('$PROJECT_SRC_DIR'), 'generated', 'esptherm_version.h') # type: ignore[name-defined] + + +def check_git(): + """Checks whether the git command is available. + + Tries to run "git --version" and exits the script if it fails. + """ + try: + subprocess.check_call(['git', '--version']) + except CalledProcessError: + print("Could not execute git. Is there a git executable on the PATH?", file=sys.stderr) + sys.exit(1) + +def get_short_hash() -> str: + """Gets the current commit short-hash. + + Runs "git rev-parse --short HEAD" to get the current git commit short-hash. + @return The current commit hash. + @raise CalledProcessError: If the command returns a non-zero exit-code. + """ + return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], text=True).strip() + + +def generate_version_header(commit: str) -> None: + """Generates the header file containing the hashes of the static files. + + Generates a header file, in src/generated, containing constant definitions with the hashes of the static files. + + @raise IOError: If opening or writing the header file fails. + """ + + print("Generating " + path.relpath(version_header_path, env.subst("$PROJECT_ROOT"))) # type: ignore[name-defined] + + with open(version_header_path, 'w') as header: + header.write( +"""/* + * esptherm_version.h + * + * **Warning:** This file is automatically generated, and should not be edited manually. + * + * This file contains version information about the ESP-WiFi-Thermometer. + * At the moment it only contains the commit hash, but once there are versioned releases it will also contain the version numbers. + * + * This project is licensed under the MIT License. + * The MIT license can be found in the project root and at https://opensource.org/licenses/MIT. + */ + +#ifndef SRC_GENERATED_ESPTHERM_VERSION_H_ +#define SRC_GENERATED_ESPTHERM_VERSION_H_ + +""") + + header.write( +f"""/** + * The short hash of the ESP-WiFi-Thermometer git commit that was used for this build. + */ +static constexpr const char *ESPTHERM_COMMIT = "{commit}"; + +""") + + header.write("#endif /* SRC_GENERATED_ESPTHERM_VERSION_H_ */" + os.linesep) + + +def main() -> int: + """The main entrypoint of this script. + + The main function generating the version header file. + + @return Zero if nothing goes wrong. + @raise IOError: If opening or reading the file fails. + """ + check_git() + commit: str = "" + try: + commit = get_short_hash() + except CalledProcessError as e: + print(f"Failed to determine git hash. Process failed with exit code {e.returncode}.", file=sys.stderr) + return 1 + + generate_version_header(commit) + + return 0 + + +if __name__ == '__main__' or __name__ == 'SCons.Script': + error: int = main() + if error != 0: + sys.exit(error) diff --git a/src/prometheus.cpp b/src/prometheus.cpp index 3b3d137..a0766cd 100644 --- a/src/prometheus.cpp +++ b/src/prometheus.cpp @@ -11,6 +11,7 @@ #include "prometheus.h" #include "main.h" #include "sensor_handler.h" +#include "generated/esptherm_version.h" #include #include #include @@ -18,7 +19,7 @@ #if ENABLE_WEB_SERVER == 1 && (ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1) std::map, uint64_t>> prom::http_requests_total; #endif -#if (ENABLE_PROMETHEUS_PUSH == 1 && ENABLE_DEEP_SLEEP_MODE != 1) +#if ENABLE_PROMETHEUS_PUSH == 1 && ENABLE_DEEP_SLEEP_MODE != 1 uint64_t prom::last_push = 0; #endif #if ENABLE_PROMETHEUS_PUSH == 1 @@ -69,26 +70,48 @@ String prom::getMetrics(const bool openmetrics) { #if ENABLE_WEB_SERVER == 1 // First determine the sum of all called path lengths. size_t uri_len_sum = 0; - for (std::pair, uint64_t>> uri_stats : http_requests_total) { + for (const std::pair, uint64_t>> &uri_stats : http_requests_total) { uri_len_sum += uri_stats.first.length(); } #endif - // The added lengths of all the lines. - // One float is assumed to have three digits before and after the dot. - // An integer is assumed to be at most 20 digits, plus four characters because of the way they are formatted. - const size_t max_len = 99 + 43 + 37 + (openmetrics ? 39 : 0) + 94 + 40 + 34 - + (openmetrics ? 36 : 0) - + PROMETHEUS_NAMESPACE_LEN * (openmetrics ? 9 : 6) + + +#if defined(ESP32) || defined(ESP8266) + const char *SDK_VERSION = ESP.getSdkVersion(); +#else + const char *SDK_VERSION = "unknown"; +#endif + const size_t SDK_VERSION_LEN = strlen(SDK_VERSION); + + // The temperature should never be more than three digits before and after the dot. + const size_t temp_max_len = 99 + 43 + PROMETHEUS_NAMESPACE_LEN * 3 + + (openmetrics ? 45 + PROMETHEUS_NAMESPACE_LEN : 0) + 38; + // The relative humidity should be three digits before and after the dot at most. + const size_t humidity_max_len = 94 + 40 + PROMETHEUS_NAMESPACE_LEN * 3 + + (openmetrics ? 42 + PROMETHEUS_NAMESPACE_LEN : 0) + 35; #ifdef ESP32 - 71 + 32 + 44 + (openmetrics ? 44 : 0) + + // A 32 bit unsigned int has 10 digits at most, plus four characters because of the way the number will be formatted. + const size_t heap_max_len = 71 + 32 + (openmetrics ? 32 : 0) + 34; +#else + const size_t heap_max_len = 0; #endif + // Assume that the hash is always seven characters long. + const size_t build_info_max_len = 73 + 25 + PROMETHEUS_NAMESPACE_LEN * 3 + + (openmetrics ? -1 : 0) + 104 + MCU_TYPE_LEN + ARDUINO_VERSION_LEN + SDK_VERSION_LEN + CPP_VERSION_LEN; #if ENABLE_WEB_SERVER == 1 - 86 + 36 + PROMETHEUS_NAMESPACE_LEN * 2 - + (83 + PROMETHEUS_NAMESPACE_LEN) * http_requests_total.size() - + uri_len_sum + + // An integer is assumed to be at most 20 digits, plus four characters because of the way they are formatted. + const size_t web_requests_total_max_len = 86 + 36 + + PROMETHEUS_NAMESPACE_LEN * 2 + + (83 + PROMETHEUS_NAMESPACE_LEN) * getRequestCounts() + + uri_len_sum; +#else + const size_t web_requests_total_max_len = 0; #endif - (openmetrics ? 5 : 0); + const size_t eof_max_len = (openmetrics ? 5 : 0); + + // The added lengths of all the lines. + const size_t max_len = temp_max_len + humidity_max_len + heap_max_len + + build_info_max_len + web_requests_total_max_len + eof_max_len; char *buffer = new char[max_len + 1]; @@ -112,6 +135,41 @@ String prom::getMetrics(const bool openmetrics) { (double) used_heap, openmetrics); #endif + len += writeMetricMetadataLine(buffer + len, "HELP", PROMETHEUS_NAMESPACE, + "build_info", "", + "A constant 1 with compile time information as labels."); + if (openmetrics) { + len += writeMetricMetadataLine(buffer + len, "TYPE", + PROMETHEUS_NAMESPACE, "build_info", "", "info"); + } else { + len += writeMetricMetadataLine(buffer + len, "TYPE", + PROMETHEUS_NAMESPACE, "build_info", "", "gauge"); + } + strcpy(buffer + len, PROMETHEUS_NAMESPACE); + len += PROMETHEUS_NAMESPACE_LEN; + strcpy(buffer + len, "_build_info{esptherm_commit=\""); + len += 29; + strcpy(buffer + len, ESPTHERM_COMMIT); + len += 7; + strcpy(buffer + len, "\",mcu_type=\""); + len += 12; + strcpy(buffer + len, MCU_TYPE); + len += MCU_TYPE_LEN; + strcpy(buffer + len, "\",arduino_version=\""); + len += 19; + strcpy(buffer + len, ARDUINO_VERSION); + len += ARDUINO_VERSION_LEN; + strcpy(buffer + len, "\",sdk_version=\""); + len += 15; + strcpy(buffer + len, SDK_VERSION); + len += SDK_VERSION_LEN; + strcpy(buffer + len, "\",cpp_std_version=\""); + len += 19; + strcpy(buffer + len, CPP_VERSION); + len += CPP_VERSION_LEN; + strcpy(buffer + len, "\"} 1\n"); + len += 5; + #if ENABLE_WEB_SERVER == 1 // Write web server statistics. len += writeMetricMetadataLine(buffer + len, "HELP", PROMETHEUS_NAMESPACE, @@ -127,41 +185,45 @@ String prom::getMetrics(const bool openmetrics) { for (std::map, uint64_t>::const_iterator response_stats = uri_stats->second.cbegin(); response_stats != uri_stats->second.cend(); response_stats++) { - strcpy(buffer + len, PROMETHEUS_NAMESPACE); + strncpy(buffer + len, PROMETHEUS_NAMESPACE, + min(max_len - len, PROMETHEUS_NAMESPACE_LEN)); len += PROMETHEUS_NAMESPACE_LEN; - strcpy(buffer + len, "_http_requests_total{method=\""); + strncpy(buffer + len, "_http_requests_total{method=\"", + min(max_len - len, (size_t) 29)); len += 29; switch (response_stats->first.first) { case HTTP_GET: - strcpy(buffer + len, "get"); + strncpy(buffer + len, "get", min(max_len - len, (size_t) 3)); len += 3; break; case HTTP_POST: - strcpy(buffer + len, "post"); + strncpy(buffer + len, "post", min(max_len - len, (size_t) 4)); len += 4; break; case HTTP_PUT: - strcpy(buffer + len, "put"); + strncpy(buffer + len, "put", min(max_len - len, (size_t) 3)); len += 3; break; case HTTP_PATCH: - strcpy(buffer + len, "patch"); + strncpy(buffer + len, "patch", min(max_len - len, (size_t) 5)); len += 5; break; case HTTP_DELETE: - strcpy(buffer + len, "delete"); + strncpy(buffer + len, "delete", min(max_len - len, (size_t) 6)); len += 6; break; case HTTP_HEAD: - strcpy(buffer + len, "head"); + strncpy(buffer + len, "head", min(max_len - len, (size_t) 4)); len += 4; break; case HTTP_OPTIONS: - strcpy(buffer + len, "options"); + strncpy(buffer + len, "options", + min(max_len - len, (size_t) 7)); len += 7; break; default: - strcpy(buffer + len, "unknown"); + strncpy(buffer + len, "unknown", + min(max_len - len, (size_t) 7)); len += 7; log_e("Unknown request method %u for uri \"%s\" in stats map.", response_stats->first.first, uri_stats->first.c_str()); @@ -180,7 +242,7 @@ String prom::getMetrics(const bool openmetrics) { } #endif /* ENABLE_WEB_SERVER == 1 */ if (openmetrics) { - strcpy(buffer + len, "# EOF\n"); + strncpy(buffer + len, "# EOF\n", min(max_len - len, (size_t) 7)); len += 6; } @@ -260,6 +322,17 @@ size_t prom::writeMetricMetadataLine(char *buffer, return written; } + +#if ENABLE_WEB_SERVER == 1 +size_t prom::getRequestCounts() { + size_t count = 0; + for (const std::pair, uint64_t>> &uri_stats : http_requests_total) { + count += uri_stats.second.size(); + } + return count; +} +#endif #endif /* ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 */ #if ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 diff --git a/src/prometheus.h b/src/prometheus.h index 6826b6a..9e7091b 100644 --- a/src/prometheus.h +++ b/src/prometheus.h @@ -33,8 +33,14 @@ * This header, and the source file with the same name, contain everything for the prometheus integration. */ namespace prom { -#if ENABLE_WEB_SERVER == 1 && (ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1) -extern std::map, uint64_t>> http_requests_total; +#if ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 +#if ENABLE_WEB_SERVER == 1 +/** + * A map used to track the number of http requests handled by this esp. + * The format is "URI -> (Method, Response Code) -> Count". + */ +extern std::map, uint64_t>> http_requests_total; #endif #if ENABLE_PROMETHEUS_PUSH == 1 #if ENABLE_DEEP_SLEEP_MODE != 1 @@ -44,6 +50,69 @@ extern AsyncClient *tcpClient; extern std::string push_url; #endif +/** + * The Hardware type this program was compiled for. + */ +#ifdef ESP32 +static constexpr const char MCU_TYPE[] = "esp32"; +#elif defined(ESP8266) +static constexpr const char MCU_TYPE[] = "esp8266"; +#else +static constexpr const char MCU_TYPE[] = "unknown"; +#endif + +/** + * The length of the microcontroller type string(MCU_TYPE). + */ +static constexpr const size_t MCU_TYPE_LEN = utils::strlen(MCU_TYPE); + +/** + * The version of the arduino implementation used on the microcontroller. + */ +#ifdef ESP32 +// ESP.getCoreVersion is relatively new, and does exactly this. +static const char ARDUINO_VERSION[] = + EXPAND_MACRO(ESP_ARDUINO_VERSION_MAJOR) "." EXPAND_MACRO(ESP_ARDUINO_VERSION_MINOR) "." EXPAND_MACRO(ESP_ARDUINO_VERSION_PATCH); +#elif defined(ESP8266) +// ESP.getCoreVersion returns an empty string for some reason. +static const char *ARDUINO_VERSION = + strcat(strcat(strcat(strcat(strcat(new char[9] { 0 }, UNSIGNED_TO_STRING(esp8266::coreVersionMajor())), + "."), UNSIGNED_TO_STRING(esp8266::coreVersionMinor())), "."), + UNSIGNED_TO_STRING(esp8266::coreVersionRevision())); +#endif + +/** + * The length of the arduino version string. + */ +static const size_t ARDUINO_VERSION_LEN = utils::strlen(ARDUINO_VERSION); + +/** + * CPP_VER is a helper macro to get the C++ version number component for the C++ standard version string. + */ +#if __cplusplus > 199711L +#define CPP_VER UNSIGNED_TO_STRING(__cplusplus % 10000 / 100) +#elif __cplusplus == 199711L +#define CPP_VER "98" +#else +#define CPP_VER "??" +#endif + +/** + * The C++ standard version used to compile this program. + */ +static const char *CPP_VERSION = +#if defined(__GNUG__) && !defined(__STRICT_ANSI__) + strcat(strcat(new char[8] { 0 }, "gnu++"), CPP_VER); +#else + strcat(strcat(new char[6] { 0 }, "c++"), CPP_VER); +#endif + +/** + * The length of the C++ standard version used to compile this program. + */ +static const size_t CPP_VERSION_LEN = utils::strlen(CPP_VERSION); +#endif /* ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 */ + /** * Initializes the prometheus integration. */ @@ -59,7 +128,7 @@ void loop(); */ void connect(); -#if (ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1) +#if ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 /** * Creates a string containing the metrics for prometheus. * @@ -116,7 +185,18 @@ template size_t writeMetricMetadataLine(char *buffer, const char (&field_name)[fnm_l], const char (&metric_namespace)[ns_l], const char (&metric_name)[nm_l], const char (&metric_unit)[u_l], const char (&value)[vl_l]); + +#if ENABLE_WEB_SERVER == 1 +/** + * Checks the total number of request counts. + * + * Used to generate the metrics page. + * + * @return The total number of metrics counts. + */ +size_t getRequestCounts(); #endif +#endif /* ENABLE_PROMETHEUS_PUSH == 1 || ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 */ #if ENABLE_PROMETHEUS_SCRAPE_SUPPORT == 1 /** diff --git a/src/webhandler.cpp b/src/webhandler.cpp index a5fc354..3d268be 100644 --- a/src/webhandler.cpp +++ b/src/webhandler.cpp @@ -126,7 +126,7 @@ bool web::csvHeaderContains(const char *header, const char *value) { && *cpos != ';') { start = cpos; } else if (start != NULL && (*cpos == ',' || *cpos == ';')) { - if (cpos - start == val_len + if ((size_t) (cpos - start) == val_len && strncmp(start, value, val_len) == 0) { return true; } @@ -137,7 +137,7 @@ bool web::csvHeaderContains(const char *header, const char *value) { } } - if (start != NULL && cpos - start == val_len + if (start != NULL && (size_t) (cpos - start) == val_len && strncmp(start, value, val_len) == 0) { return true; } else {