[Task] Update analytics db to use local.env and Remove Dynaconf from …

…Analytics (#136)

## Summary
Fixes #107 
Fixes #115 

### Time to review: __5 mins__

## Changes proposed
* removed the `*.toml` files related to dynaconf
* removed references to Dynaconf (e.g. in Docstrings, gitignore)
* use Pydantic for loading

## Context for reviewers

> After getting feedback for
#107, the consensus
was to reevaluate the way the database loader works for more uniformity.
Changes will need to be made primarily in db.py and cli.py
> 
> With the PR #84 ,
the env settings for the db are stored in settings.toml. The config
settings should be updated to use the existing local.env file

## Additional information
> Screenshots, GIF demos, code examples or output to help show the
changes working as expected.

analytics/.gitignore

@@ -1,4 +1,2 @@
 data
 
-# Ignore dynaconf secret files
-.secrets.*

analytics/config.py

@@ -1,35 +1,23 @@
-"""Loads configuration variables from settings files and settings files
+"""Loads configuration variables from settings files
 
-Dynaconf provides a few valuable features for configuration management:
-- Load variables from env vars and files with predictable overrides
-- Validate the existence and format of required configs
-- Connect with secrets managers like HashiCorp's Vault server
-- Load different configs based on environment (e.g. DEV, PROD, STAGING)
-
-For more information visit: https://www.dynaconf.com/
 """
+import os 
+from typing import Optional
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import Field
 
-from dynaconf import Dynaconf, Validator, ValidationError
+# reads environment variables from .env files defaulting to "local.env"
+class PydanticBaseEnvConfig(BaseSettings):
+    model_config = SettingsConfigDict(env_file="%s.env" % os.getenv("ENVIRONMENT", "local"), extra="ignore") # set extra to ignore so that it ignores variables irrelevant to the database config (e.g. metabase settings)
 
-settings = Dynaconf(
-    # set env vars with `export ANALYTICS_FOO=bar`
-    envvar_prefix="ANALYTICS",
-    # looks for config vars in the following files
-    # with vars in .secrets.toml overriding vars in settings.toml
-    settings_files=["settings.toml", ".secrets.toml"],
-    # merge the settings found in all files
-    merge_enabled= True,
-    # add validators for our required config vars
-    validators=[
-        Validator("SLACK_BOT_TOKEN", must_exist=True),
-        Validator("REPORTING_CHANNEL_ID", must_exist=True),
-    ],
-)
+class DBSettings(PydanticBaseEnvConfig):
+     db_host: str = Field(alias="DB_HOST")
+     port: int = Field(5432,alias="DB_PORT")
+     user: str = Field (alias="DB_USER")
+     password: str = Field(alias="DB_PASSWORD")
+     ssl_mode: str = Field(alias="DB_SSL_MODE")
+     slack_bot_token: str = Field(alias="ANALYTICS_SLACK_BOT_TOKEN")
+     reporting_channel_id: str = Field(alias="ANALYTICS_REPORTING_CHANNEL_ID")
 
-# raises after all possible errors are evaluated
-try:
-    settings.validators.validate_all()
-except ValidationError as error:
-    list_of_all_errors = error.details
-    print(list_of_all_errors)
-    raise
+def get_db_settings() -> DBSettings:
+     return DBSettings()

analytics/local.env

@@ -28,3 +28,13 @@ MB_DB_PORT=5432
 MB_DB_USER=app
 MB_DB_PASS=secret123
 MB_DB_HOST=grants-analytics-db
+
+###########################
+# Slack Configuration   #
+###########################
+# Do not add these values to this file
+# to avoid mistakenly committing them.
+# Set these in your shell
+# by doing `export ANALYTICS_REPORTING_CHANNEL_ID=whatever`
+ANALYTICS_REPORTING_CHANNEL_ID=DO_NOT_SET_HERE
+ANALYTICS_SLACK_BOT_TOKEN=DO_NOT_SET_HERE

analytics/pyproject.toml

@@ -22,6 +22,7 @@ slack-sdk = "^3.23.0"
 typer = { extras = ["all"], version = "^0.9.0" }
 sqlalchemy = "^2.0.30"
 psycopg = ">=3.0.7"
+pydantic-settings = "^2.3.4"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"

analytics/src/analytics/cli.py

@@ -241,7 +241,9 @@ def show_and_or_post_results(
     """Optionally show the results of a metric and/or post them to slack."""
     # defer load of settings until this command is called
     # this prevents an error if ANALYTICS_SLACK_BOT_TOKEN env var is unset
-    from config import settings
+    from config import get_db_settings
+
+    settings = get_db_settings()
 
     # optionally display the burndown chart in the browser
     if show_results:

analytics/src/analytics/integrations/db.py

@@ -3,13 +3,12 @@
 
 from sqlalchemy import Engine, create_engine
 
-from config import settings
+from config import get_db_settings
+
+# The variables used in the connection url are pulled from local.env
+# and configured in the DBSettings class found in config.py
 
 
-# The variables used in the connection url are set in settings.toml and
-# .secrets.toml. These can be overridden with the custom prefix defined in config.py: "ANALYTICS".
-# e.g. `export ANALYTICS_POSTGRES_USER=new_usr`.
-# Docs: https://www.dynaconf.com/envvars/
 def get_db() -> Engine:
     """
     Get a connection to the database using a SQLAlchemy engine object.
@@ -22,8 +21,10 @@ def get_db() -> Engine:
     sqlalchemy.engine.Engine
     A SQLAlchemy engine object representing the connection to the database.
     """
+    db = get_db_settings()
+    print(f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}")
     return create_engine(
-        f"postgresql+psycopg://{settings.postgres_user}:{settings.postgres_password}@{settings.postgres_host}:{settings.postgres_port}",
+        f"postgresql+psycopg://{db.user}:{db.password}@{db.db_host}:{db.port}",
         pool_pre_ping=True,
         hide_parameters=True,
     )

analytics/tests/integrations/test_slack.py

@@ -5,8 +5,9 @@
 from slack_sdk import WebClient
 
 from analytics.integrations.slack import FileMapping, SlackBot
-from config import settings
+from config import get_db_settings
 
+settings = get_db_settings()
 client = WebClient(token=settings.slack_bot_token)
 
 
@@ -19,7 +20,9 @@ def mock_slackbot() -> SlackBot:
 @pytest.mark.skip(reason="requires Slack token")
 def test_fetch_slack_channels(slackbot: SlackBot):
     """The fetch_slack_channels() function should execute correctly."""
-    result = slackbot.fetch_slack_channel_info(channel_id=settings.reporting_channel_id)
+    result = slackbot.fetch_slack_channel_info(
+        channel_id=settings.reporting_channel_id,
+    )
     assert result["ok"] is True
     assert result["channel"]["name"] == "z_bot-analytics-ci-test"