Add ClickHouse server TLS option

Signed-off-by: Yanjun Zhou <[email protected]>

build/charts/theia/README.md

@@ -37,8 +37,16 @@ Kubernetes: `>= 1.16.0-0`
 | clickhouse.monitor.image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/theia-clickhouse-monitor","tag":""}` | Container image used by the ClickHouse Monitor. |
 | clickhouse.monitor.skipRoundsNum | int | `3` | The number of rounds for the monitor to stop after a deletion to wait for the ClickHouse MergeTree Engine to release memory. |
 | clickhouse.monitor.threshold | float | `0.5` | The storage percentage at which the monitor starts to delete old records. Vary from 0 to 1. |
-| clickhouse.service.httpPort | int | `8123` | HTTP port number for the ClickHouse service. |
-| clickhouse.service.tcpPort | int | `9000` | TCP port number for the ClickHouse service. |
+| clickhouse.service.httpPort | int | `8123` | HTTP port number for ClickHouse service. |
+| clickhouse.service.secureConnection.commonName | string | `"clickhouse-clickhouse.flow-visibility.svc"` | Subject's common name. Only used when selfSignedCert is true. |
+| clickhouse.service.secureConnection.daysValid | int | `365` | Number of days for which the certificate will be valid. There is no automatic rotation with this method. This is ignored if selfSignedCert is false. |
+| clickhouse.service.secureConnection.dnsNames | list | `[]` | Optional list of alternate DNS names; may be nil. Only used when selfSignedCert is true. |
+| clickhouse.service.secureConnection.enable | bool | `false` | Determine whether to enable secure connection to ClickHouse |
+| clickhouse.service.secureConnection.httpsPort | int | `8443` | HTTP protocol TLS port. |
+| clickhouse.service.secureConnection.ipAddresses | list | `[]` | Optional list of IPs; may be nil. Only used when selfSignedCert is true. |
+| clickhouse.service.secureConnection.secureTcpPort | int | `9440` | TCP protocol TLS port. |
+| clickhouse.service.secureConnection.selfSignedCert | bool | `true` | Indicates whether to use auto-generated self-signed TLS certificates. If false, a Secret named "clickhouse-tls" must be provided with the following keys: tls.crt and tls.key. If true, the following fields commonName, ipAddresses, dnsNames, daysValid need to be provided. |
+| clickhouse.service.tcpPort | int | `9000` | TCP port number for ClickHouse service. |
 | clickhouse.service.type | string | `"ClusterIP"` | The type of Service exposing ClickHouse. It can be one of ClusterIP, NodePort or LoadBalancer. |
 | clickhouse.storage.createPersistentVolume.local.affinity | object | `{}` | Affinity for the Local PersistentVolume. By default it requires to label the Node used to store the ClickHouse data with "antrea.io/clickhouse-data-node=". |
 | clickhouse.storage.createPersistentVolume.local.nodes | list | `["kind-worker"]` | A list of Node hostnames. Required when type is "Local". Please make sure to provide (shards * replicas) Nodes. Each Node should meet affinity and have the path created on it. |

build/charts/theia/provisioning/tls/settings.xml

@@ -0,0 +1,15 @@
+<yandex>
+  <https_port>{{ .Values.clickhouse.service.secureConnection.httpsPort }}</https_port>
+  <tcp_port_secure>{{ .Values.clickhouse.service.secureConnection.secureTcpPort }}</tcp_port_secure>
+  <openSSL>
+    <server>
+      <certificateFile>/opt/certs/tls.crt</certificateFile>
+      <privateKeyFile>/opt/certs/tls.key</privateKeyFile>
+      <verificationMode>none</verificationMode>
+      <loadDefaultCAFile>true</loadDefaultCAFile>
+      <cacheSessions>true</cacheSessions>
+      <disableProtocols>sslv2,sslv3</disableProtocols>
+      <preferServerCiphers>true</preferServerCiphers>
+    </server>
+  </openSSL>
+</yandex>

build/charts/theia/templates/_helpers.tpl

@@ -42,12 +42,21 @@
 {{- $clickhouse := .clickhouse }}
 {{- $enablePV := .enablePV }}
 {{- $Chart := .Chart }}
+{{- $tls := .clickhouse.service.secureConnection }}
 - name: clickhouse
   image: {{ include "clickHouseServerImage" . | quote }}
   imagePullPolicy: {{ $clickhouse.image.pullPolicy }}
   volumeMounts:
     - name: clickhouse-configmap-volume
       mountPath: /docker-entrypoint-initdb.d
+    {{- if $tls.enable }}
+    - name: clickhouse-tls
+      mountPath: /opt/certs/tls.crt
+      subPath: tls.crt
+    - name: clickhouse-tls
+      mountPath: /opt/certs/tls.key
+      subPath: tls.key
+    {{- end }}
     {{- if not $enablePV }}
     - name: clickhouse-storage-volume
       mountPath: /var/lib/clickhouse
@@ -73,6 +82,7 @@
 
 {{- define "clickhouse.volume" }}
 {{- $clickhouse := .clickhouse }}
+{{- $tls := .clickhouse.service.secureConnection }}
 {{- $enablePV := .enablePV }}
 {{- $Files := .Files }}
 - name: clickhouse-configmap-volume
@@ -87,6 +97,12 @@
       - key: {{ regexReplaceAll "(.*)/" $path "" }}
         path: migrators/{{ regexReplaceAll "(.*)/" $path "" }}
       {{- end }}
+{{- if $tls.enable }}
+- name: clickhouse-tls
+  secret:
+    secretName: clickhouse-tls
+    optional: true
+{{- end }}
 {{- if not $enablePV }}
 - name: clickhouse-storage-volume
   emptyDir:
@@ -99,6 +115,14 @@
   name: clickhouse-monitor-coverage
 {{- end }}
 
+{{- define "clickhouse.tlsConfig" -}}
+{{- $Files := .Files }}
+{{- $Global := .Global }}
+{{- range $path, $_ :=  .Files.Glob  "provisioning/tls/*" }}
+{{ regexReplaceAll "(.*)/" $path "" }}: |
+{{ tpl ($.Files.Get $path) $Global | indent 2 }}
+{{- end }}
+{{- end -}}
 
 {{- define "theiaImageTag" -}}
 {{- $tag := .tag -}}

build/charts/theia/templates/clickhouse/clickhouseinstallation.yaml

@@ -16,8 +16,18 @@ spec:
       {{ .Values.clickhouse.connectionSecret.readOnlyUsername }}/networks/ip: "::/0"
     profiles:
       readonly/readonly: 1
+    {{- if .Values.clickhouse.service.secureConnection.enable }}
+    files:
+      {{- include "clickhouse.tlsConfig" (dict "Files" .Files "Global" .) | indent 6 }}
+    {{- end }}
     clusters:
       - name: "clickhouse"
+        {{- if .Values.clickhouse.service.secureConnection.enable }}
+        settings:
+          tcp_port_secure: {{ .Values.clickhouse.service.secureConnection.secureTcpPort }}
+          https_port: {{ .Values.clickhouse.service.secureConnection.httpsPort }}
+        secure: "yes"
+        {{- end }} 
         layout:
           {{- if gt (int .Values.clickhouse.cluster.replicas) 1 }}
           shards:
@@ -66,8 +76,16 @@ spec:
           ports:
             - name: http
               port: {{ .Values.clickhouse.service.httpPort }}
+              targetPort: 8123
             - name: tcp
               port: {{ .Values.clickhouse.service.tcpPort }}
+              targetPort: 9000
+            {{- if .Values.clickhouse.service.secureConnection.enable }}
+            - name: https
+              port: {{ .Values.clickhouse.service.secureConnection.httpsPort }}
+            - name: secureclient
+              port: {{ .Values.clickhouse.service.secureConnection.secureTcpPort }}
+            {{- end }}
     podTemplates:
       {{- if gt (int .Values.clickhouse.cluster.replicas) 1 }}
       - name: pod-template-without-monitor

build/charts/theia/templates/clickhouse/secret.yaml

@@ -1,3 +1,28 @@
+{{- $tls := .Values.clickhouse.service.secureConnection }}
+{{- if and $tls.enable $tls.selfSignedCert }}
+{{- $cert := genSelfSignedCert $tls.commonName $tls.ipAddresses (uniq (append $tls.dnsNames $tls.commonName)) ($tls.daysValid | int) }}
+{{- $certPEM := $cert.Cert | b64enc }}
+{{- $keyPEM := $cert.Key | b64enc }}
+---
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/tls
+metadata:
+  name: clickhouse-tls
+  namespace: {{ .Release.Namespace }}
+data:
+  tls.crt: {{ $certPEM | quote }}
+  tls.key: {{ $keyPEM | quote }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clickhouse-ca
+  namespace: {{ .Release.Namespace }}
+data:
+  ca.crt: {{ $certPEM | quote }}
+{{- end }}
+---
 apiVersion: v1
 kind: Secret
 metadata:

build/charts/theia/values.yaml

@@ -43,10 +43,37 @@ clickhouse:
     # -- The type of Service exposing ClickHouse. It can be one of ClusterIP,
     # NodePort or LoadBalancer.
     type: ClusterIP
-    # -- TCP port number for the ClickHouse service.
+    # -- TCP port number for ClickHouse service.
     tcpPort: 9000
-    # -- HTTP port number for the ClickHouse service.
+    # -- HTTP port number for ClickHouse service.
     httpPort: 8123
+    # As ClickHouse Operator currently does not support the cert management,
+    # we recommend to use Ingress or Load Balancer to expose ClickHouse securely.
+    # But we also provide this option to provide the secure connection from
+    # ClickHouse server end. To use this option, please rotate the certs manually.
+    secureConnection:
+      # -- Determine whether to enable secure connection to ClickHouse
+      enable: false
+      # -- TCP protocol TLS port.
+      secureTcpPort: 9440
+      # -- HTTP protocol TLS port.
+      httpsPort: 8443
+      # -- Indicates whether to use auto-generated self-signed TLS certificates.
+      # If false, a Secret named "clickhouse-tls" must be provided with the
+      # following keys: tls.crt and tls.key.
+      # If true, the following fields commonName, ipAddresses, dnsNames, daysValid
+      # need to be provided.
+      selfSignedCert: true
+      # -- Subject's common name. Only used when selfSignedCert is true.
+      commonName: "clickhouse-clickhouse.flow-visibility.svc"
+      # -- Optional list of IPs; may be nil. Only used when selfSignedCert is true.
+      ipAddresses: []
+      # -- Optional list of alternate DNS names; may be nil. Only used when
+      # selfSignedCert is true.
+      dnsNames: []
+      # -- Number of days for which the certificate will be valid. There is no automatic
+      # rotation with this method. This is ignored if selfSignedCert is false.
+      daysValid: 365
   # -- Time to live for data in the ClickHouse. Can be a plain integer using
   # one of these unit suffixes SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER,
   # YEAR.

build/yamls/flow-visibility.yml

@@ -7341,6 +7341,8 @@ spec:
         ports:
         - name: http
           port: 8123
+          targetPort: 8123
         - name: tcp
           port: 9000
+          targetPort: 9000
         type: ClusterIP

docs/network-flow-visibility.md

@@ -11,6 +11,7 @@
   - [Configuration](#configuration)
     - [With Helm](#with-helm)
       - [ClickHouse Cluster](#clickhouse-cluster)
+      - [Secure Connection](#secure-connection)
     - [With Standalone Manifest](#with-standalone-manifest)
       - [Grafana Configuration](#grafana-configuration)
         - [Service Customization](#service-customization)
@@ -289,6 +290,23 @@ ClickHouse cluster can be deployed with default Local PV or NFS PV by setting
 PV creation, you can configure a customized `StorageClass` in
 `clickhouse.storage.persistentVolumeClaimSpec`.
 
+##### Secure Connection
+
+From Theia 0.8, we support TLS connection option in ClickHouse server. To
+enable TLS connection, please set `clickhouse.service.secureConnection.enable`
+to true.
+
+Currently ClickHouse Operator does not support cert-manager, which means users
+need to manually do the certs rotation when using this ClickHouse server TLS
+option. You can set `clickhouse.service.secureConnection.selfSignedCert` to
+true to use the default certs. Please ensure that you have set the subject's
+common name. You will need to renew the certs after `clickhouse.service.secureConnection.daysValid`
+passes.
+
+You can set `clickhouse.service.secureConnection.selfSignedCert` to
+false to provide your own certificates by creating a Secret with name
+`clickhouse-tls` containing the following keys: `tls.crt` and `tls.key`.
+
 #### With Standalone Manifest
 
 If you deploy the Grafana Flow Collector with `flow-visibility.yml`, please