diff --git a/demo/app_common/api/api_common_query_postgres.py b/demo/app_common/api/api_common_query_postgres.py new file mode 100644 index 0000000..ad82708 --- /dev/null +++ b/demo/app_common/api/api_common_query_postgres.py @@ -0,0 +1,58 @@ +from typing import Optional + +import pandas as pd +from sqlalchemy import text + +from aloha.base import BaseModule +from aloha.db.postgres import PostgresOperator +from aloha.logger import LOG +from aloha.service.api.v0 import APIHandler + + +class ApiQueryPostgres(APIHandler): + def response(self, sql: str, orient: str = 'columns', config_profile: str = None, + params=None, *args, **kwargs) -> str: + op_query_db = QueryDb() + df = op_query_db.query_db(sql=sql, config_profile=config_profile, params=params) + ret = df.to_json(orient=orient, force_ascii=False) + return ret + + +class QueryDb(BaseModule): + """Read Data""" + + def get_operator(self, config_profile: str, *args, **kwargs): + config_dict = self.config[config_profile] + return PostgresOperator(config_dict) + + def query_db(self, sql: str, config_profile: str = None, params=None, *args, **kwargs) -> Optional[pd.DataFrame]: + op = self.get_operator(config_profile or 'pg_rec_readonly') + return pd.read_sql(sql=text(sql), con=op.engine, params=params) + + +default_handlers = [ + # internal API: QueryDB Postgres with sql directly + (r"/api_internal/query_postgres", ApiQueryPostgres), +] + + +def main(): + import sys + import argparse + sys.argv.pop(0) + parser = argparse.ArgumentParser() + parser.add_argument("--config-profile") + parser.add_argument("--sql", nargs='?') + args = parser.parse_args() + dict_params = vars(args) + + query = QueryDb() + op = query.get_operator(**dict_params) + LOG.info('Connection string: %s' % op.connection_str) + + if dict_params.get('sql', None) is not None: + from tabulate import tabulate + LOG.info('Query result for: %s' % dict_params['sql']) + df = query.query_db(**dict_params) + table = tabulate(df, headers='keys', tablefmt='psql') + print(table) diff --git a/demo/app_common/debug.py b/demo/app_common/debug.py index 6ac8ad0..aef06ae 100644 --- a/demo/app_common/debug.py +++ b/demo/app_common/debug.py @@ -4,7 +4,8 @@ def main(): from aloha.settings import SETTINGS modules_to_load = [ - 'app_common.api.api_common_sys_info' + "app_common.api.api_common_sys_info", + "app_common.api.api_common_query_postgres", ] if 'service' not in SETTINGS.config: diff --git a/demo/resource/config/deploy-DEV.conf b/demo/resource/config/deploy-DEV.conf new file mode 100644 index 0000000..3a361ed --- /dev/null +++ b/demo/resource/config/deploy-DEV.conf @@ -0,0 +1,9 @@ +deploy = { + postgres_db0 = { + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "postgres", + "dbname": "postgres" + } +} diff --git a/demo/resource/config/main.conf b/demo/resource/config/main.conf new file mode 100644 index 0000000..813becd --- /dev/null +++ b/demo/resource/config/main.conf @@ -0,0 +1,17 @@ +include required("deploy-DEV.conf") + +APP_MODUEL = "Aloha" + +APP_DOMAIN = { + LOCAL = "http://localhost:9999" +} + +service = { + # num_process = 1 + num_process = ${?NUM_PROCESS} + + port = ${?deploy.port_service} + port = ${?PORT_SVC} +} + +postgres_default = ${deploy.postgres_db0} diff --git a/notebook/test-api-query-postgresql.ipynb b/notebook/test-api-query-postgresql.ipynb new file mode 100644 index 0000000..b74335a --- /dev/null +++ b/notebook/test-api-query-postgresql.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Query Postgresql using API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up PYTHONPATH and DIR_RESOURCE stuff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "sys.path.insert(0, '../src/')\n", + "sys.path.insert(0, '../demo/')\n", + "os.environ['DIR_RESOURCE'] = '../demo/resource/'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Import packages and set URL endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from aloha.service.api.v0 import APICaller\n", + "\n", + "from aloha.settings import SETTINGS\n", + "\n", + "api_environment = 'LOCAL' # DEV | STG | PRD\n", + "\n", + "url_base = SETTINGS.config['APP_DOMAIN'][api_environment]\n", + "caller = APICaller(url_base)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function to query remote DB via API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "\n", + "def query_db_with_api(sql: str, config_profile='postgres_default', **kwargs):\n", + " try:\n", + " resp = caller.call('/api_internal/query_postgres', timeout=(20, 2000), data={\"sql\": sql, \"config_profile\": config_profile, })\n", + " data = resp['data']\n", + " return pd.read_json(data)\n", + " except Exception as e:\n", + " print(resp)\n", + " raise e" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simulate a time-consuming SQL query with `pg_sleep()`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "query_db_with_api(sql=\"\"\"\n", + "SELECT pg_sleep(5) AS slept\n", + "\"\"\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/notebook/test-api-service.ipynb b/notebook/test-api-service.ipynb index 6060e1c..3fee31a 100644 --- a/notebook/test-api-service.ipynb +++ b/notebook/test-api-service.ipynb @@ -7,20 +7,10 @@ "metadata": {}, "outputs": [], "source": [ - "import os, sys\n", + "import sys\n", "sys.path.insert(0, '../src/')" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "503e782e", - "metadata": {}, - "outputs": [], - "source": [ - "from aloha.service.v0 import APICaller" - ] - }, { "cell_type": "code", "execution_count": null, @@ -28,37 +18,29 @@ "metadata": {}, "outputs": [], "source": [ - "caller = APICaller()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "250d9b9c", - "metadata": {}, - "outputs": [], - "source": [ - "url_base = 'http://localhost:80'" + "from aloha.service.api.v0 import APICaller\n", + "\n", + "caller = APICaller(url_endpoint='http://localhost:9999')" ] }, { "cell_type": "code", "execution_count": null, - "id": "5d227afd", + "id": "c6d390a6", "metadata": {}, "outputs": [], "source": [ - "caller.call(url_base + '/api/common/sys_info', kind='gpu')" + "caller.call('/api/common/sys_info/gpu')" ] }, { "cell_type": "code", "execution_count": null, - "id": "c6d390a6", + "id": "351a9e14", "metadata": {}, "outputs": [], "source": [ - "caller.call(url_base + '/api/common/sys_info/gpu')" + "caller.call('/api/common/sys_info', kind='cuda')" ] } ], @@ -78,7 +60,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/notebook/test-db-postgres.ipynb b/notebook/test-db-postgres.ipynb new file mode 100644 index 0000000..834a1a0 --- /dev/null +++ b/notebook/test-db-postgres.ipynb @@ -0,0 +1,96 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! /opt/conda/bin/pip install psycopg2-binary sqlalchemy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import os, sys\n", + "sys.path.insert(0, '../src/')\n", + "os.environ['DIR_RESOURCE'] = '../demo/resource'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import aloha\n", + "from aloha.settings import SETTINGS as S\n", + "from aloha.db.postgres import PostgresOperator\n", + "\n", + "print(aloha.__path__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "op_pg = PostgresOperator(S.config.deploy.postgres_db0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "op_pg.engine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cur = op_pg.execute_query(sql=\"SELECT pg_sleep(2*5) AS slept\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "list(cur)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/src/aloha/encrypt/jwt.py b/src/aloha/encrypt/jwt.py index 33edbe6..0d5356c 100644 --- a/src/aloha/encrypt/jwt.py +++ b/src/aloha/encrypt/jwt.py @@ -2,7 +2,7 @@ from ..logger import LOG -LOG.debug('Version of pyjwt = %s' % jwt.__version__.__str__()) +LOG.debug('Using pyjwt == %s' % jwt.__version__.__str__()) def encode( diff --git a/src/aloha/service/app.py b/src/aloha/service/app.py index 57638e9..2f027ce 100644 --- a/src/aloha/service/app.py +++ b/src/aloha/service/app.py @@ -8,34 +8,34 @@ import uvloop from tornado.platform.asyncio import AsyncIOMainLoop + LOG.info('Using uvloop == %s for service event loop...' % uvloop.__version__) asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) AsyncIOMainLoop().install() - # asyncio.get_event_loop().run_forever() - # ^ the line above should be replaced by the start function below. - LOG.info('Using uvloop for service event loop...') except ImportError: - LOG.warn('[uvloop] NOT installed, fallback to asyncio loop! Please `pip install uvloop`!') + LOG.info('[uvloop] NOT installed, fallback to asyncio loop! Consider `pip install uvloop`!') from .web import WebApplication from ..settings import SETTINGS from tornado.options import options -options['log_file_prefix'] = 'access.log' - -io_loop = asyncio.get_event_loop() - class Application: def __init__(self, *args, **kwargs): + options['log_file_prefix'] = 'access.log' settings = dict(SETTINGS.config) self.web_app = WebApplication(settings) - self.io_loop = io_loop def start(self): try: self.web_app.start() - self.io_loop.run_forever() + event_loop = asyncio.get_event_loop() + if event_loop.is_running(): + # notice: the event loop MUST NOT be initialized before web_app starts (as it may fork process) + # ref: https://github.com/tornadoweb/tornado/issues/2426#issuecomment-400895086 + raise RuntimeError('Event loop already running before WebApp starts!') + else: + event_loop.run_forever() except KeyboardInterrupt: pass except Exception as e: @@ -44,4 +44,6 @@ def start(self): pass def stop(self): - self.io_loop.stop() + event_loop = asyncio.get_event_loop() + if event_loop.is_running(): + event_loop.stop() diff --git a/src/aloha/service/http/base_api_client.py b/src/aloha/service/http/base_api_client.py index 47710b4..ddc8f45 100644 --- a/src/aloha/service/http/base_api_client.py +++ b/src/aloha/service/http/base_api_client.py @@ -1,5 +1,6 @@ import uuid from abc import ABC, abstractmethod +from urllib.parse import urljoin import requests from requests.adapters import HTTPAdapter, Retry @@ -14,6 +15,10 @@ class AbstractApiClient(ABC): RETRY_STATUS_FORCELIST: frozenset = frozenset({413, 429, 503, 502, 504}) config = SETTINGS.config + def __init__(self, url_endpoint: str = None, *args, **kwargs): + self.url_endpoint = url_endpoint or '' + LOG.debug('API Caller URL endpoint set to: %s' % self.url_endpoint) + @classmethod def get_request_session(cls, total_retries: int = 3, *args, **kwargs) -> requests.Session: session = requests.Session() @@ -52,7 +57,7 @@ def call(self, api_url: str, data: dict = None, timeout=5, **kwargs): LOG.debug('Calling api: %s' % api_url) session = self.get_request_session() resp = session.post( - api_url, json=payload, timeout=timeout, headers=self.get_headers() + urljoin(self.url_endpoint, api_url), json=payload, timeout=timeout, headers=self.get_headers() ) try: diff --git a/src/aloha/service/web.py b/src/aloha/service/web.py index 07dbd52..b91072e 100644 --- a/src/aloha/service/web.py +++ b/src/aloha/service/web.py @@ -59,7 +59,7 @@ def start(self): # if overwrite port in param port = os.environ.get('port', port) - num_process = int(service_settings.get('num_process', 1)) - LOG.info('Starting service with [%s] process at port [%s]...', num_process, port) + num_process = int(service_settings.get('num_process', 0)) + LOG.info('Starting service with [%s] process at port [%s]...', num_process or 'undefined', port) self.http_server.bind(port) self.http_server.start(num_processes=num_process) diff --git a/src/aloha/util/sys_gpu.py b/src/aloha/util/sys_gpu.py index 2eae3cb..dab5186 100644 --- a/src/aloha/util/sys_gpu.py +++ b/src/aloha/util/sys_gpu.py @@ -12,7 +12,7 @@ LOG.debug('Using pynvml == %s' % nvml.__version__) except ImportError: - LOG.error('Package `pynvml` NOT installed! Cannot get GPU info.') + LOG.warn('Package `pynvml` NOT installed! Cannot get GPU info.') nvml = nvidia_smi = None Device = namedtuple('Device', field_names='index,name,arch') diff --git a/tool/run_config-aliyun.sh b/tool/run_config-aliyun.sh new file mode 100644 index 0000000..14d5556 --- /dev/null +++ b/tool/run_config-aliyun.sh @@ -0,0 +1,47 @@ +#! /bin/sh +set -ex + +export TZ=${TZ:="Asia/Shanghai"} +ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +echo "Setup timezone, current date: $(date)" + +if [ -f /etc/apt/sources.list ]; then + echo "Found Ubuntu system, setting ubuntu mirror" + sed -i 's/mirrors.*.com/mirrors.aliyun.com/' /etc/apt/sources.list + sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list + sed -i 's/security.ubuntu.com/mirrors.aliyun.com/' /etc/apt/sources.list +fi + +if [ -f /etc/yum.repos.d/CentOS-Base.repo ]; then + echo "Found CentOS system, setting CentOS mirror" + sed -i 's/mirror.centos.org/mirrors.aliyun.com/' /etc/yum.repos.d/CentOS-Base.repo + sed -i 's/mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/CentOS-Base.repo + sed -i 's/#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo +fi + +if [ -f "$(which python)" ]; then + echo "Found python, setting pypi source in /etc/pip.conf" + cat >/etc/pip.conf <>/etc/login.defs diff --git a/tool/run_config-tsinghua.sh b/tool/run_config-tsinghua.sh new file mode 100644 index 0000000..356a72f --- /dev/null +++ b/tool/run_config-tsinghua.sh @@ -0,0 +1,47 @@ +#! /bin/sh +set -ex + +export TZ=${TZ:="Asia/Shanghai"} +ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone +echo "Setup timezone, current date: $(date)" + +if [ -f /etc/apt/sources.list ]; then + echo "Found Ubuntu system, setting ubuntu mirror" + sed -i 's/mirrors.*.com/mirrors.tuna.tsinghua.edu.cn/' /etc/apt/sources.list + sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/' /etc/apt/sources.list + sed -i 's/security.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/' /etc/apt/sources.list +fi + +if [ -f /etc/yum.repos.d/CentOS-Base.repo ]; then + echo "Found CentOS system, setting CentOS mirror" + sed -i 's/mirror.centos.org/mirrors.tuna.tsinghua.edu.cn/' /etc/yum.repos.d/CentOS-Base.repo + sed -i 's/mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/CentOS-Base.repo + sed -i 's/#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo +fi + +if [ -f "$(which python)" ]; then + echo "Found python, setting pypi source in /etc/pip.conf" + cat >/etc/pip.conf <>/etc/login.defs