Skip to content

Commit

Permalink
Merge branch 'develop' into 2458-nexus-mvp
Browse files Browse the repository at this point in the history
  • Loading branch information
ADPennington authored Oct 2, 2024
2 parents 4a6ce00 + fd5cc00 commit 4e1a6f6
Show file tree
Hide file tree
Showing 14 changed files with 85 additions and 11 deletions.
11 changes: 8 additions & 3 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ version: '3'

tasks:

upload-kibana-objs:
desc: Upload dashboards to Kibana server
cmds:
- curl -X POST localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson

create-network:
desc: Create the external network
cmds:
Expand Down Expand Up @@ -29,7 +34,7 @@ tasks:
desc: Create Sentry service
dir: sentry
cmds:
# limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask : --cpuset-cpus 0
# limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask : --cpuset-cpus 0
- (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
- docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"

Expand Down Expand Up @@ -155,7 +160,7 @@ tasks:
- docker rm $(docker ps -aq) || true
- docker rmi $(docker images -q) || true
- docker volume rm $(docker volume ls -q) || true

clamav-up:
desc: Start clamav service
dir: tdrs-backend
Expand Down Expand Up @@ -187,7 +192,7 @@ tasks:
- task: frontend-up
- task: clamav-up


# need more work
frontend-init:
desc: Initialize the frontend project
Expand Down
2 changes: 1 addition & 1 deletion scripts/apply-remote-migrations.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ app=${1}
cd ./tdrs-backend

echo "Install dependencies..."
sudo apt install -y gcc
sudo apt-get install -y gcc && sudo apt-get install -y graphviz && sudo apt-get install -y graphviz-dev
sudo apt install -y libpq-dev python3-dev

python -m venv ./env
Expand Down
4 changes: 4 additions & 0 deletions scripts/deploy-backend.sh
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@ update_kibana()
cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80

# Upload dashboards to Kibana
CMD="curl -X POST $CGAPPNAME_KIBANA.apps.internal:5601/api/saved_objects/_import -H 'kbn-xsrf: true' --form file=@/home/vcap/app/tdpservice/search_indexes/kibana_saved_objs.ndjson"
cf run-task $CGAPPNAME_BACKEND --command "$CMD" --name kibana-obj-upload
}

update_backend()
Expand Down
2 changes: 2 additions & 0 deletions tdrs-backend/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ services:
- SERVER_BASEPATH=/kibana
- SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY=no-referrer
- CSP_WARNLEGACYBROWSERS=false
volumes:
- ./search_indexes/kibana_saved_objs.ndjson:/usr/share/kibana/kibana_saved_objs.ndjson
depends_on:
- elastic

Expand Down
2 changes: 1 addition & 1 deletion tdrs-backend/manifest.kibana.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ applications:
SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY: no-referrer
CSP_WARNLEGACYBROWSERS: false
docker:
image: docker.elastic.co/kibana/kibana-oss:7.4.2
image: docker.elastic.co/kibana/kibana-oss:7.10.2
command: |
export ELASTICSEARCH_HOSTS=http://$CGAPPNAME_PROXY.apps.internal:8080 &&
/usr/local/bin/dumb-init -- /usr/local/bin/kibana-docker
2 changes: 2 additions & 0 deletions tdrs-backend/tdpservice/data_files/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from io import BytesIO
import xlsxwriter
import calendar
from tdpservice.parsers.models import ParserErrorCategoryChoices


def get_xls_serialized_file(data):
Expand Down Expand Up @@ -48,6 +49,7 @@ def format_error_msg(x):
('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])),
('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])),
('row_number', lambda x: x['row_number']),
('error_type', lambda x: str(ParserErrorCategoryChoices(x['error_type']).label)),
]

# write beta banner
Expand Down
2 changes: 1 addition & 1 deletion tdrs-backend/tdpservice/parsers/test/test_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -1300,7 +1300,7 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs):
t4 = t4_objs.first()
t5 = t5_objs.last()

assert t4.CLOSURE_REASON == 8
assert t4.CLOSURE_REASON == '15'
assert t5.COUNTABLE_MONTH_FED_TIME == ' 8'


Expand Down
2 changes: 1 addition & 1 deletion tdrs-backend/tdpservice/parsers/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def transform(value, **kwargs):
month = transform_to_months(quarter)[month_index]
except ValueError:
return None
return f"{year}{month_to_int(month)}"
return int(f"{year}{month_to_int(month)}")
return transform

def tanf_ssn_decryption_func(value, **kwargs):
Expand Down
22 changes: 22 additions & 0 deletions tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Generated by Django 3.2.15 on 2024-08-29 19:21

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('search_indexes', '0030_reparse_meta_model'),
]

operations = [
migrations.AlterField(
model_name='tribal_tanf_t4',
name='CLOSURE_REASON',
field=models.CharField(max_length=2, null=True),
),
]
2 changes: 1 addition & 1 deletion tdrs-backend/tdpservice/search_indexes/models/tribal.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ class Meta:
STRATUM = models.CharField(max_length=2, null=True, blank=False)
ZIP_CODE = models.CharField(max_length=5, null=True, blank=False)
DISPOSITION = models.IntegerField(null=True, blank=False)
CLOSURE_REASON = models.IntegerField(null=True, blank=False)
CLOSURE_REASON = models.CharField(max_length=2, null=True, blank=False)
REC_SUB_HOUSING = models.IntegerField(null=True, blank=False)
REC_MED_ASSIST = models.IntegerField(null=True, blank=False)
REC_FOOD_STAMPS = models.IntegerField(null=True, blank=False)
Expand Down
13 changes: 10 additions & 3 deletions terraform/dev/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,14 @@ data "cloudfoundry_service" "elasticsearch" {
}

resource "cloudfoundry_service_instance" "elasticsearch" {
name = "es-dev"
space = data.cloudfoundry_space.space.id
service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
name = "es-dev"
space = data.cloudfoundry_space.space.id
service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
replace_on_params_change = true
json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
timeouts {
create = "60m"
update = "60m"
delete = "2h"
}
}
7 changes: 7 additions & 0 deletions terraform/production/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,11 @@ resource "cloudfoundry_service_instance" "elasticsearch" {
name = "es-prod"
space = data.cloudfoundry_space.space.id
service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-medium"]
replace_on_params_change = true
json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
timeouts {
create = "60m"
update = "60m"
delete = "2h"
}
}
7 changes: 7 additions & 0 deletions terraform/staging/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,11 @@ resource "cloudfoundry_service_instance" "elasticsearch" {
name = "es-staging"
space = data.cloudfoundry_space.space.id
service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
replace_on_params_change = true
json_params = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
timeouts {
create = "60m"
update = "60m"
delete = "2h"
}
}

0 comments on commit 4e1a6f6

Please sign in to comment.