refine web app multi-process (#16)

demo/app_common/api/api_common_query_postgres.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+import pandas as pd
+from sqlalchemy import text
+
+from aloha.base import BaseModule
+from aloha.db.postgres import PostgresOperator
+from aloha.logger import LOG
+from aloha.service.api.v0 import APIHandler
+
+
+class ApiQueryPostgres(APIHandler):
+    def response(self, sql: str, orient: str = 'columns', config_profile: str = None,
+                 params=None, *args, **kwargs) -> str:
+        op_query_db = QueryDb()
+        df = op_query_db.query_db(sql=sql, config_profile=config_profile, params=params)
+        ret = df.to_json(orient=orient, force_ascii=False)
+        return ret
+
+
+class QueryDb(BaseModule):
+    """Read Data"""
+
+    def get_operator(self, config_profile: str, *args, **kwargs):
+        config_dict = self.config[config_profile]
+        return PostgresOperator(config_dict)
+
+    def query_db(self, sql: str, config_profile: str = None, params=None, *args, **kwargs) -> Optional[pd.DataFrame]:
+        op = self.get_operator(config_profile or 'pg_rec_readonly')
+        return pd.read_sql(sql=text(sql), con=op.engine, params=params)
+
+
+default_handlers = [
+    # internal API: QueryDB Postgres with sql directly
+    (r"/api_internal/query_postgres", ApiQueryPostgres),
+]
+
+
+def main():
+    import sys
+    import argparse
+    sys.argv.pop(0)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config-profile")
+    parser.add_argument("--sql", nargs='?')
+    args = parser.parse_args()
+    dict_params = vars(args)
+
+    query = QueryDb()
+    op = query.get_operator(**dict_params)
+    LOG.info('Connection string: %s' % op.connection_str)
+
+    if dict_params.get('sql', None) is not None:
+        from tabulate import tabulate
+        LOG.info('Query result for: %s' % dict_params['sql'])
+        df = query.query_db(**dict_params)
+        table = tabulate(df, headers='keys', tablefmt='psql')
+        print(table)

demo/app_common/debug.py

@@ -4,7 +4,8 @@ def main():
     from aloha.settings import SETTINGS
 
     modules_to_load = [
-        'app_common.api.api_common_sys_info'
+        "app_common.api.api_common_sys_info",
+        "app_common.api.api_common_query_postgres",
     ]
 
     if 'service' not in SETTINGS.config:

demo/resource/config/deploy-DEV.conf

@@ -0,0 +1,9 @@
+deploy = {
+  postgres_db0 = {
+    "host": "localhost",
+    "port": 5432,
+    "user": "postgres",
+    "password": "postgres",
+    "dbname": "postgres"
+  }
+}

demo/resource/config/main.conf

@@ -0,0 +1,17 @@
+include required("deploy-DEV.conf")
+
+APP_MODUEL = "Aloha"
+
+APP_DOMAIN = {
+  LOCAL = "http://localhost:9999"
+}
+
+service = {
+  # num_process = 1
+  num_process = ${?NUM_PROCESS}
+
+  port = ${?deploy.port_service}
+  port = ${?PORT_SVC}
+}
+
+postgres_default = ${deploy.postgres_db0}

notebook/test-api-query-postgresql.ipynb

@@ -0,0 +1,120 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Query Postgresql using API"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Setting up PYTHONPATH and DIR_RESOURCE stuff"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "\n",
+    "sys.path.insert(0, '../src/')\n",
+    "sys.path.insert(0, '../demo/')\n",
+    "os.environ['DIR_RESOURCE'] = '../demo/resource/'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Import packages and set URL endpoint"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from aloha.service.api.v0 import APICaller\n",
+    "\n",
+    "from aloha.settings import SETTINGS\n",
+    "\n",
+    "api_environment = 'LOCAL'  # DEV | STG | PRD\n",
+    "\n",
+    "url_base = SETTINGS.config['APP_DOMAIN'][api_environment]\n",
+    "caller = APICaller(url_base)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Function to query remote DB via API"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "\n",
+    "def query_db_with_api(sql: str, config_profile='postgres_default', **kwargs):\n",
+    "    try:\n",
+    "        resp = caller.call('/api_internal/query_postgres', timeout=(20, 2000), data={\"sql\": sql, \"config_profile\": config_profile, })\n",
+    "        data = resp['data']\n",
+    "        return pd.read_json(data)\n",
+    "    except Exception as e:\n",
+    "        print(resp)\n",
+    "        raise e"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Simulate a time-consuming SQL query with `pg_sleep()`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_db_with_api(sql=\"\"\"\n",
+    "SELECT pg_sleep(5) AS slept\n",
+    "\"\"\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

notebook/test-api-service.ipynb

@@ -7,58 +7,40 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os, sys\n",
+    "import sys\n",
     "sys.path.insert(0, '../src/')"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "503e782e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from aloha.service.v0 import APICaller"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "12967663",
    "metadata": {},
    "outputs": [],
    "source": [
-    "caller = APICaller()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "250d9b9c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "url_base = 'http://localhost:80'"
+    "from aloha.service.api.v0 import APICaller\n",
+    "\n",
+    "caller = APICaller(url_endpoint='http://localhost:9999')"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5d227afd",
+   "id": "c6d390a6",
    "metadata": {},
    "outputs": [],
    "source": [
-    "caller.call(url_base + '/api/common/sys_info', kind='gpu')"
+    "caller.call('/api/common/sys_info/gpu')"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "c6d390a6",
+   "id": "351a9e14",
    "metadata": {},
    "outputs": [],
    "source": [
-    "caller.call(url_base + '/api/common/sys_info/gpu')"
+    "caller.call('/api/common/sys_info', kind='cuda')"
    ]
   }
  ],
@@ -78,7 +60,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.2"
+   "version": "3.10.6"
   }
  },
  "nbformat": 4,

notebook/test-db-postgres.ipynb

@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! /opt/conda/bin/pip install psycopg2-binary sqlalchemy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import os, sys\n",
+    "sys.path.insert(0, '../src/')\n",
+    "os.environ['DIR_RESOURCE'] = '../demo/resource'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import aloha\n",
+    "from aloha.settings import SETTINGS as S\n",
+    "from aloha.db.postgres import PostgresOperator\n",
+    "\n",
+    "print(aloha.__path__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "op_pg = PostgresOperator(S.config.deploy.postgres_db0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "op_pg.engine"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cur = op_pg.execute_query(sql=\"SELECT pg_sleep(2*5) AS slept\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "list(cur)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

src/aloha/encrypt/jwt.py

@@ -2,7 +2,7 @@
 
 from ..logger import LOG
 
-LOG.debug('Version of pyjwt = %s' % jwt.__version__.__str__())
+LOG.debug('Using pyjwt == %s' % jwt.__version__.__str__())
 
 
 def encode(