Merge pull request #3 from ustudio/update-for-new-cluster-style

updated to get information from kubernetes API instead of downward API and support one namespace per application.

circle.yml

@@ -1,6 +1,6 @@
 machine:
   post:
-    - pyenv global 2.7.11 3.5.2
+    - pyenv global 2.7.11 3.5.2 3.6.0
 dependencies:
   override:
     - pip install tox

setup.py

@@ -6,12 +6,12 @@
 
 install_requires = [
     "datadog-logger",
-    "kubernetes-downward-api",
-    "datadog"
+    "datadog",
+    "kubernetes"
 ]
 
 setup(name="ustack-logging",
-      version="0.1.0",
+      version="0.2.0",
       description="Default logging configuration for uStack style Python applications.",
       url="https://github.com/ustudio/ustack-logging",
       packages=["ustack_logging"],

tests/test_ustack_logging.py

@@ -1,6 +1,9 @@
+import base64
 import logging
 import unittest
 
+from kubernetes.client import V1Secret, V1Pod, V1ObjectMeta
+
 try:
     from unittest import mock
 except ImportError:
@@ -10,55 +13,64 @@
 
 
 class TestLogging(unittest.TestCase):
+    @mock.patch("socket.gethostname")
+    @mock.patch("kubernetes.config.load_incluster_config")
+    @mock.patch("kubernetes.client.CoreV1Api")
     @mock.patch("ustack_logging.logging_configuration.log_error_events", autospec=True)
-    @mock.patch("kubernetes_downward_api.parse", autospec=True)
     @mock.patch("datadog.initialize", autospec=True)
     @mock.patch("logging.basicConfig", autospec=True)
     def test_configures_logging_format_and_logs_errors_to_datadog(
-            self, mock_log_config, mock_dd_init, mock_k8s_parse, mock_log_errors):
-        mock_k8s_parse.return_value = {
-            "namespace": "dev",
-            "labels": {
-                "app": "my_app",
-                "role": "app"
-            }
-        }
-
-        configure_logging({
-            "DATADOG_API_KEY": "dd-api-key",
-            "DATADOG_APP_KEY": "dd-app-key"
-        })
+            self, mock_log_config, mock_dd_init, mock_log_errors, mock_k8s_api_class,
+            mock_k8s_config, mock_gethostname):
+
+        mock_k8s_api = mock_k8s_api_class.return_value
+
+        mock_gethostname.return_value = "podname.domain"
+
+        mock_k8s_api.read_namespaced_secret.return_value = V1Secret(
+            data={
+                "environment": base64.b64encode("dev".encode("utf8")),
+                "datadog-api-key": base64.b64encode("dd-api-key".encode("utf8")),
+                "datadog-app-key": base64.b64encode("dd-app-key".encode("utf8"))
+            })
+        mock_k8s_api.read_namespaced_pod.return_value = V1Pod(
+            metadata=V1ObjectMeta(labels={
+                "role": "my-role"
+            }))
+
+        with mock.patch(
+                "ustack_logging.logging_configuration.open",
+                mock.mock_open(read_data="my-app"),
+                create=True) as mock_open:
+            configure_logging()
 
         mock_log_config.assert_called_once_with(
             format="%(asctime)s %(levelname)s:%(module)s:%(message)s",
             datefmt="%Y-%m-%d %H:%M:%S%z", level=logging.INFO)
 
+        mock_k8s_config.assert_called_once_with()
+
+        mock_k8s_api.read_namespaced_secret.assert_called_once_with(
+            "environment-info", "ustudio-system")
+
         mock_dd_init.assert_called_once_with(api_key="dd-api-key", app_key="dd-app-key")
 
-        mock_k8s_parse.assert_called_once_with(["/etc/podinfo"])
+        mock_open.assert_called_once_with("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+
+        mock_gethostname.assert_called_once_with()
+
+        mock_k8s_api.read_namespaced_pod.assert_called_once_with("podname", "my-app")
+
         mock_log_errors.assert_called_once_with(
-            tags=["environment:dev", "service:my_app", "role:app"])
+            tags=["environment:dev", "service:my-app", "role:my-role"])
 
-    @mock.patch("ustack_logging.logging_configuration.log_error_events", autospec=True)
-    @mock.patch("kubernetes_downward_api.parse", autospec=True)
-    @mock.patch("datadog.initialize", autospec=True)
+    @mock.patch("kubernetes.config.load_incluster_config")
     @mock.patch("logging.basicConfig", autospec=True)
-    def test_ignores_errors_from_datadog_initialization(
-            self, mock_log_config, mock_dd_init, mock_k8s_parse, mock_log_errors):
-        mock_k8s_parse.return_value = {
-            "namespace": "dev",
-            "labels": {
-                "app": "my_app",
-                "role": "app"
-            }
-        }
-
-        mock_dd_init.side_effect = RuntimeError
-
-        configure_logging({
-            "DATADOG_API_KEY": "dd-api-key",
-            "DATADOG_APP_KEY": "dd-app-key"
-        })
+    def test_ignores_errors_from_datadog_initialization(self, mock_log_config, mock_k8s_config):
+
+        mock_k8s_config.side_effect = RuntimeError
+
+        configure_logging()
 
         mock_log_config.assert_called_once_with(
             format="%(asctime)s %(levelname)s:%(module)s:%(message)s",

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py35
+envlist = py27, py35, py36
 
 [testenv]
 commands = nosetests

ustack_logging/logging_configuration.py

@@ -1,23 +1,38 @@
+import base64
 import datadog
 from datadog_logger import log_error_events
-import kubernetes_downward_api
+import kubernetes.client
 import logging
+import socket
 
 
-def configure_logging(environ):
+def configure_logging():
     logging.basicConfig(
         format="%(asctime)s %(levelname)s:%(module)s:%(message)s",
         datefmt="%Y-%m-%d %H:%M:%S%z", level=logging.INFO)
 
     try:
-        datadog.initialize(api_key=environ["DATADOG_API_KEY"], app_key=environ["DATADOG_APP_KEY"])
+        kubernetes.config.load_incluster_config()
 
-        podinfo = kubernetes_downward_api.parse(["/etc/podinfo"])
+        core_api = kubernetes.client.CoreV1Api()
+        environment_info = core_api.read_namespaced_secret("environment-info", "ustudio-system")
+
+        datadog.initialize(
+            api_key=base64.b64decode(environment_info.data["datadog-api-key"]).decode("utf8"),
+            app_key=base64.b64decode(environment_info.data["datadog-app-key"]).decode("utf8"))
+
+        with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f:
+            namespace = f.read()
+
+        pod_name = socket.gethostname().split(".")[0]
+
+        pod_info = core_api.read_namespaced_pod(pod_name, namespace)
 
         log_error_events(tags=[
-            "environment:{0}".format(podinfo["namespace"]),
-            "service:{0}".format(podinfo["labels"]["app"]),
-            "role:{0}".format(podinfo["labels"]["role"])
+            "environment:{0}".format(
+                base64.b64decode(environment_info.data["environment"]).decode("utf8")),
+            "service:{0}".format(namespace),
+            "role:{0}".format(pod_info.metadata.labels["role"])
         ])
     except:
         logging.warning("Could not initialize DataDog error logging, with error:", exc_info=True)