From 2a8e9d09719e56e3397a5df76d833cad924061e1 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 25 Jul 2019 12:55:50 +0530 Subject: [PATCH 01/11] changes to upgrade to Sharq core 0.5.1 for clustered redis support --- requirements.txt | 3 +-- setup.py | 2 +- src/config/sharq.conf.ctmpl | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5d36821..df61ef2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Flask==0.10.1 Jinja2==2.7.2 MarkupSafe==0.23 -SharQ==0.4.0 +SharQ==0.5.1 Werkzeug==0.9.4 argparse==1.2.1 gevent==1.0.1 @@ -9,6 +9,5 @@ greenlet==0.4.2 gunicorn==19.0.0 itsdangerous==0.24 msgpack-python==0.4.2 -redis==2.10.1 ujson==1.33 wsgiref==0.1.2 diff --git a/setup.py b/setup.py index 8333dad..30c1e26 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ packages=['sharq_server'], py_modules=['runner'], install_requires=[ - 'SharQ==0.4.0', + 'SharQ==0.5.1', 'Flask==0.10.1', 'Jinja2==2.7.2', 'MarkupSafe==0.23', diff --git a/src/config/sharq.conf.ctmpl b/src/config/sharq.conf.ctmpl index 2e6d83a..d6e71ca 100644 --- a/src/config/sharq.conf.ctmpl +++ b/src/config/sharq.conf.ctmpl @@ -11,7 +11,7 @@ accesslog : /tmp/sharq.log ; optional [redis] db : 0 -key_prefix : sharq_server +key_prefix : {{ printf "%s/%s/%s/config/redis/key_prefix" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} conn_type : tcp_sock ; or tcp_sock ;; unix connection settings ;; unix_socket_path : /var/run/redis/redis.sock @@ -19,3 +19,4 @@ conn_type : tcp_sock ; or tcp_sock port : {{ printf "%s/%s/%s/config/redis/port" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key | parseInt }} host : {{ printf "%s/%s/%s/config/redis/host" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} +clustered : {{ printf "%s/%s/%s/config/redis/clustered" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} From bdf4bc5fee02655165d726cb5307bb178ee9acb4 Mon Sep 17 00:00:00 2001 From: Tarun Date: Thu, 25 Jul 2019 17:40:59 +0530 Subject: [PATCH 02/11] changes for sharq core lib --- .gitignore | 1 + ci/Dockerfile | 1 + requirements.txt | 1 - setup.py | 1 - sharq.conf | 1 + src/config/sharq.conf.ctmpl | 2 +- 6 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5696078..d752e61 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ Dockerfile_supervisor test_sharq.py .idea/* venv/* +.vscode/ diff --git a/ci/Dockerfile b/ci/Dockerfile index 5f85864..c12dce8 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y nginx && pip install virtualenv envtpl RUN curl -L https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tgz | tar -C /usr/sbin -xzf - RUN virtualenv /opt/sharq-server RUN . /opt/sharq-server/bin/activate && /opt/sharq-server/bin/pip install --no-cache-dir -r /opt/sharq-server/requirements.txt && /opt/sharq-server/bin/python setup.py install -f && /opt/sharq-server/bin/pip install uwsgi +RUN /opt/sharq-server/bin/pip install https://sharq-dist.s3.amazonaws.com/SharQ-0.5.2.tar.gz ADD src/config /etc/sharq-server/config ADD src/config/nginx.conf /etc/nginx/nginx.conf diff --git a/requirements.txt b/requirements.txt index df61ef2..075a41a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ Flask==0.10.1 Jinja2==2.7.2 MarkupSafe==0.23 -SharQ==0.5.1 Werkzeug==0.9.4 argparse==1.2.1 gevent==1.0.1 diff --git a/setup.py b/setup.py index 30c1e26..5f6eb04 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ packages=['sharq_server'], py_modules=['runner'], install_requires=[ - 'SharQ==0.5.1', 'Flask==0.10.1', 'Jinja2==2.7.2', 'MarkupSafe==0.23', diff --git a/sharq.conf b/sharq.conf index 978b6fc..86891d8 100644 --- a/sharq.conf +++ b/sharq.conf @@ -18,3 +18,4 @@ unix_socket_path : /tmp/redis.sock ;; tcp connection settings port : 6379 host : 127.0.0.1 +clustered : true diff --git a/src/config/sharq.conf.ctmpl b/src/config/sharq.conf.ctmpl index d6e71ca..3eb8030 100644 --- a/src/config/sharq.conf.ctmpl +++ b/src/config/sharq.conf.ctmpl @@ -19,4 +19,4 @@ conn_type : tcp_sock ; or tcp_sock port : {{ printf "%s/%s/%s/config/redis/port" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key | parseInt }} host : {{ printf "%s/%s/%s/config/redis/host" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} -clustered : {{ printf "%s/%s/%s/config/redis/clustered" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} +clustered : {{ printf "%s/%s/%s/config/redis/clustered" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} From 202ca292b9c5399026afe7e0c8333add71c45209 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 17 Dec 2019 17:19:38 +0530 Subject: [PATCH 03/11] multiregion sharq config --- src/config/sharq.conf.ctmpl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/config/sharq.conf.ctmpl b/src/config/sharq.conf.ctmpl index 3eb8030..735696f 100644 --- a/src/config/sharq.conf.ctmpl +++ b/src/config/sharq.conf.ctmpl @@ -1,3 +1,7 @@ +{{$region := env "REGION"}} +{{$appenv := env "ENVIRONMENT"}} +{{$team := env "TEAM"}} +{{$sharq_type := env "SHARQ_TYPE"}} [sharq] job_expire_interval : 45000 ; in milliseconds job_requeue_interval : 1000 ; in milliseconds @@ -11,12 +15,12 @@ accesslog : /tmp/sharq.log ; optional [redis] db : 0 -key_prefix : {{ printf "%s/%s/%s/config/redis/key_prefix" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} +key_prefix : {{ printf "%s/%s/%s/config/redis/key_prefix" $team $appenv $sharq_type $region | key }} conn_type : tcp_sock ; or tcp_sock ;; unix connection settings ;; unix_socket_path : /var/run/redis/redis.sock ;; tcp connection settings -port : {{ printf "%s/%s/%s/config/redis/port" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key | parseInt }} -host : {{ printf "%s/%s/%s/config/redis/host" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} -clustered : {{ printf "%s/%s/%s/config/redis/clustered" (env "TEAM") (env "ENVIRONMENT") (env "SHARQ_TYPE") | key }} +port : {{ printf "%s/%s/%s/%s/config/redis/port" $team $appenv $sharq_type $region | key | parseInt }} +host : {{ printf "%s/%s/%s/%s/config/redis/host" $team $appenv $sharq_type $region | key }} +clustered : {{ printf "%s/%s/%s/%s/config/redis/clustered" $team $appenv $sharq_type $region | key }} From 36321357ceef7d01b81868a43e632c20603b1c8f Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Fri, 20 Dec 2019 11:08:46 +0530 Subject: [PATCH 04/11] add missing path in config --- src/config/sharq.conf.ctmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/sharq.conf.ctmpl b/src/config/sharq.conf.ctmpl index 735696f..a316369 100644 --- a/src/config/sharq.conf.ctmpl +++ b/src/config/sharq.conf.ctmpl @@ -15,7 +15,7 @@ accesslog : /tmp/sharq.log ; optional [redis] db : 0 -key_prefix : {{ printf "%s/%s/%s/config/redis/key_prefix" $team $appenv $sharq_type $region | key }} +key_prefix : {{ printf "%s/%s/%s/%s/config/redis/key_prefix" $team $appenv $sharq_type $region | key }} conn_type : tcp_sock ; or tcp_sock ;; unix connection settings ;; unix_socket_path : /var/run/redis/redis.sock From 7019951d64bb75c3a82c4db0ff9c500db7a1d5a4 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 24 Dec 2019 12:51:28 +0530 Subject: [PATCH 05/11] add deep status --- sharq_server/server.py | 9 ++++++++- src/config/nginx-sharq.conf | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index 1201a18..23d94c7 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -53,6 +53,9 @@ def __init__(self, config_path): self.app.add_url_rule( '/deletequeue///', view_func=self._view_clear_queue, methods=['DELETE']) + self.app.add_url_rule( + '/deepstatus/', + view_func=self._view_deep_status(), methods=['GET']) def requeue(self): """Loop endlessly and requeue expired jobs.""" @@ -176,7 +179,11 @@ def _view_metrics(self, queue_type, queue_id): return jsonify(**response), 400 return jsonify(**response) - + + def _view_deep_status(self): + print "Deep status check" # TODO remove this + self.sq.ping() + def _view_clear_queue(self, queue_type, queue_id): """remove queueu from SharQ based on the queue_type and queue_id.""" response = { diff --git a/src/config/nginx-sharq.conf b/src/config/nginx-sharq.conf index 4067f84..f4d81dd 100644 --- a/src/config/nginx-sharq.conf +++ b/src/config/nginx-sharq.conf @@ -10,6 +10,12 @@ server { include uwsgi_params; } + location /deepstatus/ { + log_not_found off; + uwsgi_pass unix:///var/run/sharq/sharq.sock; + include uwsgi_params; + } + location / { # Not needed because it's all in the VPC log_not_found off; From 82866d4de422fd7fe4523babb789ed0f371b5c94 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 24 Dec 2019 13:05:48 +0530 Subject: [PATCH 06/11] change sharq core image --- ci/Dockerfile | 2 +- sharq_server/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index c12dce8..b86fa4e 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y nginx && pip install virtualenv envtpl RUN curl -L https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tgz | tar -C /usr/sbin -xzf - RUN virtualenv /opt/sharq-server RUN . /opt/sharq-server/bin/activate && /opt/sharq-server/bin/pip install --no-cache-dir -r /opt/sharq-server/requirements.txt && /opt/sharq-server/bin/python setup.py install -f && /opt/sharq-server/bin/pip install uwsgi -RUN /opt/sharq-server/bin/pip install https://sharq-dist.s3.amazonaws.com/SharQ-0.5.2.tar.gz +RUN /opt/sharq-server/bin/pip install https://voice-sharq.s3.amazonaws.com/SharQ-0.5.2.tar.gz ADD src/config /etc/sharq-server/config ADD src/config/nginx.conf /etc/nginx/nginx.conf diff --git a/sharq_server/server.py b/sharq_server/server.py index 23d94c7..cb11a71 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -185,7 +185,7 @@ def _view_deep_status(self): self.sq.ping() def _view_clear_queue(self, queue_type, queue_id): - """remove queueu from SharQ based on the queue_type and queue_id.""" + """remove queue from SharQ based on the queue_type and queue_id.""" response = { 'status': 'failure' } From e1b6c30af91bee606bb8f7fd3e0cb8182aebc070 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 24 Dec 2019 15:04:55 +0530 Subject: [PATCH 07/11] fix bug --- sharq_server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index cb11a71..ee15ea4 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -55,7 +55,7 @@ def __init__(self, config_path): view_func=self._view_clear_queue, methods=['DELETE']) self.app.add_url_rule( '/deepstatus/', - view_func=self._view_deep_status(), methods=['GET']) + view_func=self._view_deep_status, methods=['GET']) def requeue(self): """Loop endlessly and requeue expired jobs.""" From 08d83929d9d0aba2b0612440d7463f62ff66c437 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 24 Dec 2019 15:34:24 +0530 Subject: [PATCH 08/11] return proper response --- sharq_server/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index ee15ea4..0cdf674 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -181,8 +181,11 @@ def _view_metrics(self, queue_type, queue_id): return jsonify(**response) def _view_deep_status(self): - print "Deep status check" # TODO remove this - self.sq.ping() + """Checks underlying data store health""" + response = { + 'status': self.sq.ping() + } + return jsonify(**response) def _view_clear_queue(self, queue_type, queue_id): """remove queue from SharQ based on the queue_type and queue_id.""" From 9ba313522d8a11941a1109c4ac324101f430528b Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 24 Dec 2019 15:41:25 +0530 Subject: [PATCH 09/11] update response message --- sharq_server/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sharq_server/server.py b/sharq_server/server.py index 0cdf674..a10547c 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -182,8 +182,9 @@ def _view_metrics(self, queue_type, queue_id): def _view_deep_status(self): """Checks underlying data store health""" + self.sq.ping() response = { - 'status': self.sq.ping() + 'status': "success" } return jsonify(**response) From f0482e0423888e9ce5ceba5f86e94670db5ed6ef Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Tue, 7 Jan 2020 12:57:33 +0530 Subject: [PATCH 10/11] add debug logs --- sharq_server/server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sharq_server/server.py b/sharq_server/server.py index a10547c..f8a207a 100644 --- a/sharq_server/server.py +++ b/sharq_server/server.py @@ -107,6 +107,10 @@ def _view_dequeue(self, queue_type): if response['status'] == 'failure': return jsonify(**response), 404 except Exception, e: + print e + import traceback + for line in traceback.format_exc().splitlines(): + print line response['message'] = e.message return jsonify(**response), 400 From aea3e6d202f0dc94ba5d141084fda243c733bd16 Mon Sep 17 00:00:00 2001 From: Manish Chand Kaushik Date: Wed, 19 Feb 2020 18:53:00 +0530 Subject: [PATCH 11/11] removing local bucket adding public release of SharQ core lib --- ci/Dockerfile | 1 - requirements.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index b86fa4e..5f85864 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y nginx && pip install virtualenv envtpl RUN curl -L https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tgz | tar -C /usr/sbin -xzf - RUN virtualenv /opt/sharq-server RUN . /opt/sharq-server/bin/activate && /opt/sharq-server/bin/pip install --no-cache-dir -r /opt/sharq-server/requirements.txt && /opt/sharq-server/bin/python setup.py install -f && /opt/sharq-server/bin/pip install uwsgi -RUN /opt/sharq-server/bin/pip install https://voice-sharq.s3.amazonaws.com/SharQ-0.5.2.tar.gz ADD src/config /etc/sharq-server/config ADD src/config/nginx.conf /etc/nginx/nginx.conf diff --git a/requirements.txt b/requirements.txt index 075a41a..9beced4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ itsdangerous==0.24 msgpack-python==0.4.2 ujson==1.33 wsgiref==0.1.2 +SharQ==0.5.2