From f8c174dbe04d98e7de8939d07986bc98ee423076 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 13:53:38 +0200 Subject: [PATCH 1/6] Add CRD deployment mode to infra collector Signed-off-by: Matej Gera --- .../k8s-helm/values-crd.yaml | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 otel-infrastructure-collector/k8s-helm/values-crd.yaml diff --git a/otel-infrastructure-collector/k8s-helm/values-crd.yaml b/otel-infrastructure-collector/k8s-helm/values-crd.yaml new file mode 100644 index 00000000..d395076c --- /dev/null +++ b/otel-infrastructure-collector/k8s-helm/values-crd.yaml @@ -0,0 +1,182 @@ +global: + domain: "" + traces: + endpoint: "" + metrics: + endpoint: "" + logs: + endpoint: "" + defaultApplicationName: "default" + defaultSubsystemName: "nodes" + +opentelemetry-collector: + mode: deployment + collectorCRD: + generate: true + fullnameOverride: otel-infrastructure-collector + configMap: + create: false + clusterRole: + name: "otel-infrastructure-collector" + create: true + rules: + - apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["watch", "list"] + clusterRoleBinding: + name: "otel-infrastructure-collector" + replicaCount: 1 + presets: + clusterMetrics: + enabled: true + kubernetesAttributes: + enabled: true + mysql: + metrics: + enabled: false + instances: + - username: "" + password: "" + port: 3306 + extraLogs: + enabled: false + volumeMountName: "" + mountPath: "" + + ports: + otlp: + enabled: true + otlp-http: + enabled: false + jaeger-compact: + enabled: false + jaeger-thrift: + enabled: false + jaeger-grpc: + enabled: false + zipkin: + enabled: false + + extraEnvs: + - name: CORALOGIX_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: coralogix-keys + key: PRIVATE_KEY + config: + extensions: + zpages: + endpoint: localhost:55679 + receivers: + k8sobjects: + objects: + - name: events + mode: pull + interval: 15s + group: events.k8s.io + prometheus: + config: + scrape_configs: + - job_name: opentelemetry-infrastructure-collector + scrape_interval: 30s + static_configs: + - targets: + - ${MY_POD_IP}:8888 + exporters: + coralogix: + timeout: "1m" + private_key: "${CORALOGIX_PRIVATE_KEY}" + domain: "{{.Values.global.domain}}" + traces: + endpoint: "{{ .Values.global.traces.endpoint }}" + metrics: + endpoint: "{{ .Values.global.metrics.endpoint }}" + logs: + endpoint: "{{ .Values.global.logs.endpoint }}" + application_name_attributes: + - "k8s.namespace.name" + - "service.namespace" + subsystem_name_attributes: + - "k8s.deployment.name" + - "k8s.statefulset.name" + - "k8s.daemonset.name" + - "k8s.cronjob.name" + - "k8s.job.name" + - "k8s.container.name" + - "k8s.node.name" + - "service.name" + application_name: "{{.Values.global.defaultApplicationName }}" + subsystem_name: "{{.Values.global.defaultSubsystemName }}" + processors: + memory_limiter: null # Will get the k8s resource limits + transform/kube-events: + log_statements: + - context: log + statements: + - keep_keys(body, ["type", "action", "eventTime", "reason", "regarding", "reportingController", "note", "series", "metadata", "deprecatedFirstTimestamp", "deprecatedLastTimestamp"]) + resource/kube-events: + attributes: + - key: service.name + value: "kube-events" + action: upsert + service: + extensions: + - zpages + - health_check + - memory_ballast + telemetry: + logs: + encoding: json + metrics: + address: ${MY_POD_IP}:8888 + pipelines: + logs/kube-events: + exporters: + - coralogix + processors: + - memory_limiter + - batch + - transform/kube-events + - resource/kube-events + receivers: + - k8sobjects + metrics: + exporters: + - coralogix + processors: + - memory_limiter + - batch + receivers: + - prometheus + - otlp + logs: + exporters: + - coralogix + processors: + - memory_limiter + - batch + receivers: + - otlp + tolerations: + - operator: Exists + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 1 + memory: 2G + + # In order to enable serviceMonitor, following part must be enabled in order to expose the required port: + # ports: + # metrics: + # enabled: true + + # serviceMonitor: + # enabled: true + + # prometheusRule: + # enabled: true + # defaultRules: + # enabled: true From cc1185a882eb8d26db0e6d89a4310927597810be Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 13:54:11 +0200 Subject: [PATCH 2/6] Add MySQL preset to values file; enable k8s attributes Signed-off-by: Matej Gera --- otel-infrastructure-collector/k8s-helm/values.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/otel-infrastructure-collector/k8s-helm/values.yaml b/otel-infrastructure-collector/k8s-helm/values.yaml index 7d897bca..59c3beef 100644 --- a/otel-infrastructure-collector/k8s-helm/values.yaml +++ b/otel-infrastructure-collector/k8s-helm/values.yaml @@ -25,6 +25,20 @@ opentelemetry-collector: presets: clusterMetrics: enabled: true + kubernetesAttributes: + enabled: true + mysql: + metrics: + enabled: false + instances: + - username: "" + password: "" + port: 3306 + extraLogs: + enabled: false + volumeMountName: "" + mountPath: "" + ports: otlp: enabled: true From c0f0921409e311e07510aa69f1194c0ddb97314a Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 13:54:34 +0200 Subject: [PATCH 3/6] Update README with instructions on CRD and MySQL preset Signed-off-by: Matej Gera --- .../k8s-helm/README.md | 122 +++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/otel-infrastructure-collector/k8s-helm/README.md b/otel-infrastructure-collector/k8s-helm/README.md index cb483567..454feb57 100644 --- a/otel-infrastructure-collector/k8s-helm/README.md +++ b/otel-infrastructure-collector/k8s-helm/README.md @@ -4,6 +4,13 @@ This Infrastructure collector provides: - [Coralogix Exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/coralogixexporter) - Coralogix exporter is preconfigured to enrich data using Kubernetes Attributes, which allows quick correlation of telemetry signals using consistent ApplicationName and SubsytemName fields. - [Cluster Metrics Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver) - The Kubernetes Cluster receiver collects cluster-level metrics from the Kubernetes API server. Alternative to Kube State Metrics project. +- [Integrations presets](#integration-presets) - This chart provides support to integrate with various applications running on your cluster to monitor them out of the box. + +### OpenTelemetry Operator (for CRD users) + +If you wish to use this Helm chart as an `OpenTelemetryCollector` CRD, you will need to have the OpenTelemetry Operator installed in your cluster. Please refer to the [OpenTelemetry Operator documentation](https://github.com/open-telemetry/opentelemetry-operator/blob/main/README.md) for full details. + +We recommend to install the operator with the help of the community Helm charts from the [OpenTelemetry Helm Charts](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-operator) repository. ### Required @@ -52,6 +59,31 @@ helm upgrade --install otel-infrastructure-collector coralogix-charts-virtual/ot -f values.yaml ``` +### Generating OpenTelemetryCollector CRD for OpenTelemetry Operator users + +If you wish to deploy the `otel-agent` using the OpenTelemetry Operator, you can generate an `OpenTelemetryCollector` CRD. You might want to do this if you'd like to take advantage of some advanced features provided by the operator, such as automatic collector upgrade or CRD-defined auto-instrumentation. + +For full details on how to install and use the operator, please refer to the [OpenTelemetry Operator documentation](https://github.com/open-telemetry/opentelemetry-operator/blob/main/README.md). + +First make sure to add our Helm charts repository to the local repos list with the following command: + +```bash +helm repo add coralogix-charts-virtual https://cgx.jfrog.io/artifactory/coralogix-charts-virtual +``` + +In order to get the updated Helm charts from the added repository, please run: + +```bash +helm repo update +``` + +Install the chart with the CRD `values.yaml` file: + +```bash +helm upgrade --install otel-coralogix-agent coralogix-charts-virtual/opentelemetry-coralogix \ + -f values-crd.yaml +``` + # Infrastructure Monitoring ## Kubernetes Events @@ -81,11 +113,99 @@ processors: ``` This configuration is filtering out any event that has the field `reason` with one of those values `BackoffLimitExceeded|FailedScheduling|Unhealthy`, for more information about the `filter` processor feel free to check the official documentation [here](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor). - ## Cluster Receiver ## Alerts +# Integration presets + +The `otel-infrastructure-collector` chart also provides support to integrate with different applications. The following integration presets are available. + +## MySQL + +The MySQL preset is able to collect metrics and extra logs (slow query log, general query log) from your MySQL instances. **Extra logs collection is available only when running the `otel-infrastructure-collector` as CRD with the OpenTelemetry Operator.** + +### Prerequisites + +This preset supports MySQL version 8.0 + +Collecting most metrics requires the ability of the database user to execute `SHOW GLOBAL STATUS`. + +### Configuration for metrics collection + +The metrics collection has to be enabled by setting the `metrics.enabled` to `true`. + +Each MySQL instance is configured in the `metrics.instances` section. You can configure multiple instances, if you have more than one instance you'd like to monitor. + +Required instance settings: +- `username`: The username of the database user that will be used to collect metrics. +- `password`: The password of the database user that will be used to collect metrics. We strongly recommend to provide this via a Kuberetes secret as an environment variable, e.g `MYSQL_PASSWORD`, which should be provided in the `extraEnv` section of the chart. This parameter should be passed in format `${env:MYSQL_PASSWORD}` in order for the collector to be able to read it. + +Optional instance settings: +- `port`: The port of the MySQL instance. Defaults to `3306`. Unless you use non-standard port, there is no need to set this parameter. +- `labelSelectors`: A list of label selectors to select the pods that run the MySQL instances. If you wish to monitor mutiple instance, the selectors will determine which pods belong to a given instance. + +### Configuration for extra logs collection + +The extra logs collection has to be enabled by setting the `extraLogs.enabled` to `true`. Note that the extra logs have to enabled on your MySQL instance (please refer to https://dev.mysql.com/doc/refman/8.0/en/server-logs.html). Please also note that extra logs collection is only available when running `otel-infrastructure-collector` with OpenTelemetry Operator. + +**PLEASE NOTE:** In order for the collection to take effect, you need to annotate your MySQL instance(s) pod templates with the following: + +```bash +kubectl patch sts -p '{"spec": {"template":{"metadata":{"annotations":{"sidecar.opentelemetry.io/inject":"otel-infrastructure-collector-mysql-logs-sidecar"}}}} }' +``` + +Required settings: +- `volumeMountName`: specifies the name of the volume mount. It should correspond to the volume name of the MySQL data volume. +- `mountPath`: specifies the path at which to mount the volume. This should correspond the mount path of your MySQL data volume. Provide this parameter without trailing slash. + +Optional settings: +- `logFilesPath`: specifies which directory to watch for log files. This will typically be the MySQL data directory, + such as `/var/lib/mysql`. If not specified, the value of `mountPath` will be used. +- `logFilesExtension`: specifies which file extensions to watch for. Defaults to `.log`. + +### Common issues + +- Metrics collection is failing with error `"Error 1227 (42000): Access denied; you need (at least one of) the PROCESS privilege(s) for this operation"` + - This error indicates that the database user you provided does not have the required privileges to collect metrics. Provide the `PROCESS` privilege to the user, e.g. by running query + `GRANT PROCESS ON *.* TO 'user'@'%'` + +### Example preset configuration for single instance + +```yaml + mysql: + metrics: + enabled: true + instances: + - username: "otel-coralogix-collector" + password: ${env:MYSQL_PASSWORD} + extraLogs: + enabled: true + volumeMountName: "data" + mountPath: "/var/log/mysql" +``` + +### Example preset configuration for multiple instance + +```yaml + mysql: + metrics: + enabled: true + instances: + - username: "otel-coralogix-collector" + password: ${env:MYSQL_PASSWORD_INSTANCE_A} + labelSelectors: + app.kubernetes.io/name: "mysql-a" + - username: "otel-coralogix-collector" + password: ${env:MYSQL_PASSWORD_INSTANCE_B} + labelSelectors: + app.kubernetes.io/name: "mysql-b" + extraLogs: + enabled: true + volumeMountName: "data" + mountPath: "/var/log/mysql" +``` + # Dependencies This chart uses [openetelemetry-collector](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) helm chart. From 9bcf0f4c450617669bc692160ca8eb0c7e643485 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 13:57:45 +0200 Subject: [PATCH 4/6] Use fork as dependency Signed-off-by: Matej Gera --- otel-infrastructure-collector/k8s-helm/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/otel-infrastructure-collector/k8s-helm/Chart.yaml b/otel-infrastructure-collector/k8s-helm/Chart.yaml index 8200abe0..706289b2 100644 --- a/otel-infrastructure-collector/k8s-helm/Chart.yaml +++ b/otel-infrastructure-collector/k8s-helm/Chart.yaml @@ -8,8 +8,8 @@ keywords: - Coralogix dependencies: - name: opentelemetry-collector - version: "0.55.0" - repository: https://open-telemetry.github.io/opentelemetry-helm-charts + version: "0.63.0" + repository: https://cgx.jfrog.io/artifactory/coralogix-charts-virtual sources: - https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector maintainers: From dcdba8de205bdb2b1b8f7f6fd38bd4a575e70792 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 13:58:00 +0200 Subject: [PATCH 5/6] Bump chart and add changelog Signed-off-by: Matej Gera --- otel-infrastructure-collector/CHANGELOG.md | 7 +++++++ otel-infrastructure-collector/k8s-helm/Chart.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/otel-infrastructure-collector/CHANGELOG.md b/otel-infrastructure-collector/CHANGELOG.md index 15dfdce5..308eb9dd 100644 --- a/otel-infrastructure-collector/CHANGELOG.md +++ b/otel-infrastructure-collector/CHANGELOG.md @@ -2,6 +2,13 @@ ## OpenTelemtry-Infrastructure-Collector +### v0.1.3 / 2023-07-17 + +* [FEATURE] Add support for deploying `otel-infrastructure-collector` with OpenTelemetry Operator +* [FEATURE] Add MySQL preset for metrics and extra logs +* [CHORE] Update OpenTelemetry Collector to v0.77.0 +* [CHORE] Use Coralogix fork for OpenTelemetry Collector Helm chart dependency + ### v0.1.2 / 2023-05-08 * [FEATURE] Allow users to configure Coralogix domain instead of endpoints diff --git a/otel-infrastructure-collector/k8s-helm/Chart.yaml b/otel-infrastructure-collector/k8s-helm/Chart.yaml index 706289b2..bf1e95a9 100644 --- a/otel-infrastructure-collector/k8s-helm/Chart.yaml +++ b/otel-infrastructure-collector/k8s-helm/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: otel-infrastructure-collector description: OpenTelemetry Infrastructure collector -version: 0.1.2 +version: 0.1.3 keywords: - OpenTelemetry Collector - OpenTelemetry Infrastructure Collector From bcb6277785ae8c16febd1091d23eefe9dae55af9 Mon Sep 17 00:00:00 2001 From: Matej Gera Date: Mon, 17 Jul 2023 14:06:30 +0200 Subject: [PATCH 6/6] Fix docs Signed-off-by: Matej Gera --- otel-infrastructure-collector/k8s-helm/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/otel-infrastructure-collector/k8s-helm/README.md b/otel-infrastructure-collector/k8s-helm/README.md index 454feb57..133d0037 100644 --- a/otel-infrastructure-collector/k8s-helm/README.md +++ b/otel-infrastructure-collector/k8s-helm/README.md @@ -113,6 +113,7 @@ processors: ``` This configuration is filtering out any event that has the field `reason` with one of those values `BackoffLimitExceeded|FailedScheduling|Unhealthy`, for more information about the `filter` processor feel free to check the official documentation [here](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor). + ## Cluster Receiver ## Alerts @@ -139,7 +140,7 @@ Each MySQL instance is configured in the `metrics.instances` section. You can co Required instance settings: - `username`: The username of the database user that will be used to collect metrics. -- `password`: The password of the database user that will be used to collect metrics. We strongly recommend to provide this via a Kuberetes secret as an environment variable, e.g `MYSQL_PASSWORD`, which should be provided in the `extraEnv` section of the chart. This parameter should be passed in format `${env:MYSQL_PASSWORD}` in order for the collector to be able to read it. +- `password`: The password of the database user that will be used to collect metrics. We strongly recommend to provide this via a Kuberetes secret as an environment variable, e.g `MYSQL_PASSWORD`, which should be provided in the `extraEnv` section of the chart. This parameter should be passed in format `${env:MYSQL_PASSWORD}` in order for the collector to be able to read it. Optional instance settings: - `port`: The port of the MySQL instance. Defaults to `3306`. Unless you use non-standard port, there is no need to set this parameter. @@ -147,7 +148,7 @@ Optional instance settings: ### Configuration for extra logs collection -The extra logs collection has to be enabled by setting the `extraLogs.enabled` to `true`. Note that the extra logs have to enabled on your MySQL instance (please refer to https://dev.mysql.com/doc/refman/8.0/en/server-logs.html). Please also note that extra logs collection is only available when running `otel-infrastructure-collector` with OpenTelemetry Operator. +The extra logs collection has to be enabled by setting the `extraLogs.enabled` to `true`. Note that the extra logs have to enabled on your MySQL instance (please refer to [relevant documentation](https://dev.mysql.com/doc/refman/8.0/en/server-logs.html)). Please also note that extra logs collection is only available when running `otel-infrastructure-collector` with OpenTelemetry Operator. **PLEASE NOTE:** In order for the collection to take effect, you need to annotate your MySQL instance(s) pod templates with the following: @@ -161,14 +162,14 @@ Required settings: Optional settings: - `logFilesPath`: specifies which directory to watch for log files. This will typically be the MySQL data directory, - such as `/var/lib/mysql`. If not specified, the value of `mountPath` will be used. + such as `/var/lib/mysql`. If not specified, the value of `mountPath` will be used. - `logFilesExtension`: specifies which file extensions to watch for. Defaults to `.log`. ### Common issues - Metrics collection is failing with error `"Error 1227 (42000): Access denied; you need (at least one of) the PROCESS privilege(s) for this operation"` - - This error indicates that the database user you provided does not have the required privileges to collect metrics. Provide the `PROCESS` privilege to the user, e.g. by running query - `GRANT PROCESS ON *.* TO 'user'@'%'` + - This error indicates that the database user you provided does not have the required privileges to collect metrics. Provide the `PROCESS` privilege to the user, e.g. by running query + `GRANT PROCESS ON *.* TO 'user'@'%'` ### Example preset configuration for single instance