Feature/tags for elasticsearchwriter

AUTHORS

@@ -255,6 +255,7 @@ Sascha Westermann <[email protected]>
 Sebastian Brückner <[email protected]>
 Sebastian Chrostek <[email protected]>
 Sebastian Eikenberg <[email protected]>
+Sebastian Grund <[email protected]>
 Sebastian Marsching <[email protected]>
 Silas <[email protected]>
 Simon Murray <[email protected]>

doc/09-object-types.md

@@ -1181,7 +1181,7 @@ Configuration Attributes:
 
 ### ElasticsearchWriter <a id="objecttype-elasticsearchwriter"></a>
 
-Writes check result metrics and performance data to an Elasticsearch instance.
+Writes check result metrics and performance data to an Elasticsearch or OpenSearch instance.
 This configuration object is available as [elasticsearch feature](14-features.md#elasticsearch-writer).
 
 Example:
@@ -1194,6 +1194,10 @@ object ElasticsearchWriter "elasticsearch" {
 
   enable_send_perfdata = true
 
+  host_tags_template = {
+    os_name = "$host.vars.os$"
+  }
+
   flush_threshold = 1024
   flush_interval = 10
 }
@@ -1215,6 +1219,8 @@ Configuration Attributes:
   password                  | String                | **Optional.** Basic auth password if Elasticsearch is hidden behind an HTTP proxy.
   enable\_tls               | Boolean               | **Optional.** Whether to use a TLS stream. Defaults to `false`. Requires an HTTP proxy.
   insecure\_noverify        | Boolean               | **Optional.** Disable TLS peer verification.
+  host\_tags\_template      | Dictionary            | **Optional.** Allows to apply additional tags to the Elasticsearch host entries. They will be added under `tags`.
+  service\_tags\_template   | Dictionary            | **Optional.** Allows to apply additional tags to the Elasticsearch service entries. They will be added under `tags`.
   ca\_path                  | String                | **Optional.** Path to CA certificate to validate the remote host. Requires `enable_tls` set to `true`.
   cert\_path                | String                | **Optional.** Path to host certificate to present to the remote host for mutual verification. Requires `enable_tls` set to `true`.
   key\_path                 | String                | **Optional.** Path to host key to accompany the cert\_path. Requires `enable_tls` set to `true`.

doc/14-features.md

@@ -335,16 +335,14 @@ More integrations:
 #### Elasticsearch Writer <a id="elasticsearch-writer"></a>
 
 This feature forwards check results, state changes and notification events
-to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) installation over its HTTP API.
+to an [Elasticsearch](https://www.elastic.co/products/elasticsearch) or an [OpenSearch](https://opensearch.org/) installation over its HTTP API.
 
 The check results include parsed performance data metrics if enabled.
 
 > **Note**
 >
-> Elasticsearch 5.x or 6.x are required. This feature has been successfully tested with
-> Elasticsearch 5.6.7 and 6.3.1.
-
-
+> Elasticsearch 7.x, 8.x or Opensearch 2.12.x are required. This feature has been successfully tested with
+> Elasticsearch 7.17.10, 8.8.1 and OpenSearch 2.13.0.
 
 Enable the feature and restart Icinga 2.
 
@@ -398,6 +396,24 @@ check_result.perfdata.<perfdata-label>.warn
 check_result.perfdata.<perfdata-label>.crit
 ```
 
+Additionaly it is possible to configure custom tags that are applied to the metrics via `host_tags_template` or `service_tags_template`.
+Depending on whether the write event was triggered on a service or host object, additional tags are added to the ElasticSearch entries.
+
+A host metrics entry configured with the following `host_tags_template`:
+```
+host_tags_template = {
+  os_name = "$host.vars.os$"
+  custom_label = "A Custom Label"
+  a_list_of_items = [ "foo", "bar" ]
+}
+```
+Will in addition to the above mentioned lines also contain:
+```
+tags.os_name = "Linux"
+tags.custom_label = "A Custom Label"
+tags.a_list_of_items = [ "foo", "bar" ]
+```
+
 #### Elasticsearch in Cluster HA Zones <a id="elasticsearch-writer-cluster-ha"></a>
 
 The Elasticsearch feature supports [high availability](06-distributed-monitoring.md#distributed-monitoring-high-availability-features)

lib/perfdata/elasticsearchwriter.cpp

@@ -5,6 +5,7 @@
 #include "remote/url.hpp"
 #include "icinga/compatutility.hpp"
 #include "icinga/service.hpp"
+#include "icinga/macroprocessor.hpp"
 #include "icinga/checkcommand.hpp"
 #include "base/application.hpp"
 #include "base/defer.hpp"
@@ -131,6 +132,40 @@ void ElasticsearchWriter::Pause()
 	ObjectImpl<ElasticsearchWriter>::Pause();
 }
 
+void ElasticsearchWriter::AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
+{
+	Host::Ptr host;
+	Service::Ptr service;
+	tie(host, service) = GetHostService(checkable);
+
+	Dictionary::Ptr tmpl;
+	if (service && GetServiceTagsTemplate()) {
+		tmpl = static_pointer_cast<Dictionary>(GetServiceTagsTemplate()->ShallowClone());
+	} else if (GetHostTagsTemplate()) {
+		tmpl = static_pointer_cast<Dictionary>(GetHostTagsTemplate()->ShallowClone());
+	}
+
+	if (tmpl) {
+		MacroProcessor::ResolverList resolvers;
+		resolvers.emplace_back("host", host);
+		if (service) {
+			resolvers.emplace_back("service", service);
+		}
+
+		Dictionary::Ptr tags = new Dictionary();
+		ObjectLock olock(tmpl);
+		for (const Dictionary::Pair& pair : tmpl) {
+			String missingMacro;
+			Value value = MacroProcessor::ResolveMacros(pair.second, resolvers, cr, &missingMacro);
+
+			if (missingMacro.IsEmpty()) {
+				tags->Set(pair.first, value);
+			}
+		}
+		fields->Set("tags", tags);
+	}
+}
+
 void ElasticsearchWriter::AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr)
 {
 	String prefix = "check_result.";
@@ -257,6 +292,8 @@ void ElasticsearchWriter::InternalCheckResultHandler(const Checkable::Ptr& check
 		ts = cr->GetExecutionEnd();
 	}
 
+	AddTemplateTags(fields, checkable, cr);
+
 	Enqueue(checkable, "checkresult", fields, ts);
 }
 
@@ -307,6 +344,8 @@ void ElasticsearchWriter::StateChangeHandlerInternal(const Checkable::Ptr& check
 		ts = cr->GetExecutionEnd();
 	}
 
+	AddTemplateTags(fields, checkable, cr);
+
 	Enqueue(checkable, "statechange", fields, ts);
 }
 
@@ -377,6 +416,8 @@ void ElasticsearchWriter::NotificationSentToAllUsersHandlerInternal(const Notifi
 		ts = cr->GetExecutionEnd();
 	}
 
+	AddTemplateTags(fields, checkable, cr);
+
 	Enqueue(checkable, "notification", fields, ts);
 }
 
@@ -683,3 +724,49 @@ String ElasticsearchWriter::FormatTimestamp(double ts)
 
 	return Utility::FormatDateTime("%Y-%m-%dT%H:%M:%S", ts) + "." + Convert::ToString(milliSeconds) + Utility::FormatDateTime("%z", ts);
 }
+
+void ElasticsearchWriter::ValidateHostTagsTemplate(const Lazy<Dictionary::Ptr>& lvalue, const ValidationUtils& utils)
+{
+	ObjectImpl<ElasticsearchWriter>::ValidateHostTagsTemplate(lvalue, utils);
+
+	Dictionary::Ptr tags = lvalue();
+	if (tags) {
+		ObjectLock olock(tags);
+		for (const Dictionary::Pair& pair : tags) {
+			if (pair.second.IsObjectType<Array>()) {
+				Array::Ptr arrObject = pair.second;
+				ObjectLock arrLock(arrObject);
+				for (const Value& arrValue : arrObject) {
+					if (!MacroProcessor::ValidateMacroString(arrValue)) {
+						BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'."));
+					}
+				}
+			} else if (!MacroProcessor::ValidateMacroString(pair.second)) {
+				BOOST_THROW_EXCEPTION(ValidationError(this, { "host_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'."));
+			}
+		}
+	}
+}
+
+void ElasticsearchWriter::ValidateServiceTagsTemplate(const Lazy<Dictionary::Ptr>& lvalue, const ValidationUtils& utils)
+{
+	ObjectImpl<ElasticsearchWriter>::ValidateServiceTagsTemplate(lvalue, utils);
+
+	Dictionary::Ptr tags = lvalue();
+	if (tags) {
+		ObjectLock olock(tags);
+		for (const Dictionary::Pair& pair : tags) {
+			if (pair.second.IsObjectType<Array>()) {
+				Array::Ptr arrObject = pair.second;
+				ObjectLock arrLock(arrObject);
+				for (const Value& arrValue : arrObject) {
+					if (!MacroProcessor::ValidateMacroString(arrValue)) {
+						BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + arrValue + "'."));
+					}
+				}
+			} else if (!MacroProcessor::ValidateMacroString(pair.second)) {
+				BOOST_THROW_EXCEPTION(ValidationError(this, { "service_tags_template", pair.first }, "Closing $ not found in macro format string '" + pair.second + "'."));
+			}
+		}
+	}
+}

lib/perfdata/elasticsearchwriter.hpp

@@ -23,6 +23,9 @@ class ElasticsearchWriter final : public ObjectImpl<ElasticsearchWriter>
 
 	static String FormatTimestamp(double ts);
 
+	void ValidateHostTagsTemplate(const Lazy<Dictionary::Ptr> &lvalue, const ValidationUtils &utils) override;
+	void ValidateServiceTagsTemplate(const Lazy<Dictionary::Ptr> &lvalue, const ValidationUtils &utils) override;
+
 protected:
 	void OnConfigLoaded() override;
 	void Resume() override;
@@ -37,6 +40,7 @@ class ElasticsearchWriter final : public ObjectImpl<ElasticsearchWriter>
 	std::mutex m_DataBufferMutex;
 
 	void AddCheckResult(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
+	void AddTemplateTags(const Dictionary::Ptr& fields, const Checkable::Ptr& checkable, const CheckResult::Ptr& cr);
 
 	void StateChangeHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type);
 	void StateChangeHandlerInternal(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, StateType type);

lib/perfdata/elasticsearchwriter.ti

@@ -29,6 +29,9 @@ class ElasticsearchWriter : ConfigObject
 	[config] bool enable_tls {
 		default {{{ return false; }}}
 	};
+
+	[config] Dictionary::Ptr host_tags_template;
+	[config] Dictionary::Ptr service_tags_template;
 	[config] bool insecure_noverify {
 		default {{{ return false; }}}
 	};
@@ -47,4 +50,19 @@ class ElasticsearchWriter : ConfigObject
 	};
 };
 
+validator ElasticsearchWriter {
+	Dictionary host_tags_template {
+		String "*";
+		Array "*" {
+			String "*";
+		};
+	};
+	Dictionary service_tags_template {
+		String "*";
+		Array "*" {
+			String "*";
+		};
+	};
+};
+
 }