Reparse Memory Management

tdrs-backend/tdpservice/conftest.py

@@ -395,6 +395,12 @@ def test_private_key():
     yield get_private_key(key)
 
 
+@pytest.fixture()
+def system_user():
+    """Create system user."""
+    return UserFactory.create(username='system')
+
+
 # Register factories with pytest-factoryboy for automatic dependency injection
 # of model-related fixtures into tests.
 register(OwaspZapScanFactory)

tdrs-backend/tdpservice/parsers/parse.py

@@ -211,11 +211,10 @@ def rollback_records(unsaved_records, datafile):
                                  f"Encountered error while indexing datafile documents: \n{e}",
                                  "error"
                                  )
-            logger.warn("Encountered an Elastic exception, enforcing DB cleanup.")
+            logger.warning("Encountered an Elastic exception, enforcing DB cleanup.")
             num_deleted, models = qset.delete()
-            logger.info("Succesfully performed DB cleanup after elastic failure.")
             log_parser_exception(datafile,
-                                 "Succesfully performed DB cleanup after elastic failure.",
+                                 "Succesfully performed DB cleanup after elastic failure in rollback_records.",
                                  "info"
                                  )
         except DatabaseError as e:
@@ -310,7 +309,7 @@ def delete_serialized_records(duplicate_manager, dfs):
             total_deleted += num_deleted
             dfs.total_number_of_records_created -= num_deleted
             log_parser_exception(dfs.datafile,
-                                 "Succesfully performed DB cleanup after elastic failure.",
+                                 "Succesfully performed DB cleanup after elastic failure in delete_serialized_records.",
                                  "info"
                                  )
         except DatabaseError as e:

tdrs-backend/tdpservice/parsers/test/factories.py

@@ -184,43 +184,43 @@ class Meta:
     EMPLOYMENT_STATUS = 1
     WORK_ELIGIBLE_INDICATOR = "01"
     WORK_PART_STATUS = "01"
-    UNSUB_EMPLOYMENT = 1
-    SUB_PRIVATE_EMPLOYMENT = 1
-    SUB_PUBLIC_EMPLOYMENT = 1
-    WORK_EXPERIENCE_HOP = 1
-    WORK_EXPERIENCE_EA = 1
-    WORK_EXPERIENCE_HOL = 1
-    OJT = 1
-    JOB_SEARCH_HOP = 1
-    JOB_SEARCH_EA = 1
-    JOB_SEARCH_HOL = 1
-    COMM_SERVICES_HOP = 1
-    COMM_SERVICES_EA = 1
-    COMM_SERVICES_HOL = 1
-    VOCATIONAL_ED_TRAINING_HOP = 1
-    VOCATIONAL_ED_TRAINING_EA = 1
-    VOCATIONAL_ED_TRAINING_HOL = 1
-    JOB_SKILLS_TRAINING_HOP = 1
-    JOB_SKILLS_TRAINING_EA = 1
-    JOB_SKILLS_TRAINING_HOL = 1
-    ED_NO_HIGH_SCHOOL_DIPL_HOP = 1
-    ED_NO_HIGH_SCHOOL_DIPL_EA = 1
-    ED_NO_HIGH_SCHOOL_DIPL_HOL = 1
-    SCHOOL_ATTENDENCE_HOP = 1
-    SCHOOL_ATTENDENCE_EA = 1
-    SCHOOL_ATTENDENCE_HOL = 1
-    PROVIDE_CC_HOP = 1
-    PROVIDE_CC_EA = 1
-    PROVIDE_CC_HOL = 1
-    OTHER_WORK_ACTIVITIES = 1
-    DEEMED_HOURS_FOR_OVERALL = 1
-    DEEMED_HOURS_FOR_TWO_PARENT = 1
-    EARNED_INCOME = 1
-    UNEARNED_INCOME_TAX_CREDIT = 1
-    UNEARNED_SOCIAL_SECURITY = 1
-    UNEARNED_SSI = 1
-    UNEARNED_WORKERS_COMP = 1
-    OTHER_UNEARNED_INCOME = 1
+    UNSUB_EMPLOYMENT = "01"
+    SUB_PRIVATE_EMPLOYMENT = "01"
+    SUB_PUBLIC_EMPLOYMENT = "01"
+    WORK_EXPERIENCE_HOP = "01"
+    WORK_EXPERIENCE_EA = "01"
+    WORK_EXPERIENCE_HOL = "01"
+    OJT = "01"
+    JOB_SEARCH_HOP = "01"
+    JOB_SEARCH_EA = "01"
+    JOB_SEARCH_HOL = "01"
+    COMM_SERVICES_HOP = "01"
+    COMM_SERVICES_EA = "01"
+    COMM_SERVICES_HOL = "01"
+    VOCATIONAL_ED_TRAINING_HOP = "01"
+    VOCATIONAL_ED_TRAINING_EA = "01"
+    VOCATIONAL_ED_TRAINING_HOL = "01"
+    JOB_SKILLS_TRAINING_HOP = "01"
+    JOB_SKILLS_TRAINING_EA = "01"
+    JOB_SKILLS_TRAINING_HOL = "01"
+    ED_NO_HIGH_SCHOOL_DIPL_HOP = "01"
+    ED_NO_HIGH_SCHOOL_DIPL_EA = "01"
+    ED_NO_HIGH_SCHOOL_DIPL_HOL = "01"
+    SCHOOL_ATTENDENCE_HOP = "01"
+    SCHOOL_ATTENDENCE_EA = "01"
+    SCHOOL_ATTENDENCE_HOL = "01"
+    PROVIDE_CC_HOP = "01"
+    PROVIDE_CC_EA = "01"
+    PROVIDE_CC_HOL = "01"
+    OTHER_WORK_ACTIVITIES = "01"
+    DEEMED_HOURS_FOR_OVERALL = "01"
+    DEEMED_HOURS_FOR_TWO_PARENT = "01"
+    EARNED_INCOME = "01"
+    UNEARNED_INCOME_TAX_CREDIT = "01"
+    UNEARNED_SOCIAL_SECURITY = "01"
+    UNEARNED_SSI = "01"
+    UNEARNED_WORKERS_COMP = "01"
+    OTHER_UNEARNED_INCOME = "01"
 
 
 class TanfT3Factory(factory.django.DjangoModelFactory):
@@ -451,10 +451,10 @@ class Meta:
     CURRENT_MONTH_STATE_EXEMPT = 1
     EMPLOYMENT_STATUS = 1
     WORK_PART_STATUS = "01"
-    UNSUB_EMPLOYMENT = 1
-    SUB_PRIVATE_EMPLOYMENT = 1
-    SUB_PUBLIC_EMPLOYMENT = 1
-    OJT = 1
+    UNSUB_EMPLOYMENT = "01"
+    SUB_PRIVATE_EMPLOYMENT = "01"
+    SUB_PUBLIC_EMPLOYMENT = "01"
+    OJT = "01"
     JOB_SEARCH = '1'
     COMM_SERVICES = '1'
     VOCATIONAL_ED_TRAINING = '1'

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -2,6 +2,7 @@
 
 
 import pytest
+import os
 from django.contrib.admin.models import LogEntry
 from django.conf import settings
 from django.db.models import Q as Query
@@ -1739,6 +1740,9 @@ def test_parse_duplicate(file, batch_size, model, record_type, num_errors, dfs,
     settings.BULK_CREATE_BATCH_SIZE = batch_size
 
     parse.parse_datafile(datafile, dfs)
+
+    settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000)
+
     parser_errors = ParserError.objects.filter(file=datafile,
                                                error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by('id')
     for e in parser_errors:
@@ -1782,6 +1786,9 @@ def test_parse_partial_duplicate(file, batch_size, model, record_type, num_error
     settings.BULK_CREATE_BATCH_SIZE = batch_size
 
     parse.parse_datafile(datafile, dfs)
+
+    settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000)
+
     parser_errors = ParserError.objects.filter(file=datafile,
                                                error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by('id')
     for e in parser_errors:
@@ -1806,6 +1813,8 @@ def test_parse_cat_4_edge_case_file(cat4_edge_case_file, dfs):
 
     parse.parse_datafile(cat4_edge_case_file, dfs)
 
+    settings.BULK_CREATE_BATCH_SIZE = os.getenv("BULK_CREATE_BATCH_SIZE", 10000)
+
     parser_errors = ParserError.objects.filter(file=cat4_edge_case_file).filter(
         error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY)
 

tdrs-backend/tdpservice/parsers/validators/category3.py

@@ -386,7 +386,7 @@ def validate(record, row_schema):
                 "Caught exception in validator: validate__WORK_ELIGIBLE_INDICATOR__HOH__AGE. " +
                 f"With field values: {vals}."
             )
-            logger.error(f'Exception: {e}')
+            logger.debug(f'Exception: {e}')
             # Per conversation with Alex on 03/26/2024, returning the true case during exception handling to avoid
             # confusing the STTs.
             return true_case

tdrs-backend/tdpservice/scheduling/management/db_backup.py

@@ -20,7 +20,10 @@
 
 
 OS_ENV = os.environ
-content_type = ContentType.objects.get_for_model(LogEntry)
+
+def get_content_type():
+    """Get content type for log entry."""
+    return ContentType.objects.get_for_model(LogEntry)
 
 def get_system_values():
     """Return dict of keys and settings to use whether local or deployed."""
@@ -91,7 +94,7 @@ def backup_database(file_name,
         logger.info(msg)
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Executed Database Backup",
             action_flag=ADDITION,
@@ -123,7 +126,7 @@ def restore_database(file_name, postgres_client, database_uri, system_user):
         msg = "Completed database creation."
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Executed Database create",
             action_flag=ADDITION,
@@ -145,7 +148,7 @@ def restore_database(file_name, postgres_client, database_uri, system_user):
         msg = "Completed database restoration."
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Executed Database restore",
             action_flag=ADDITION,
@@ -177,7 +180,7 @@ def upload_file(file_name, bucket, sys_values, system_user, object_name=None, re
     msg = "Successfully uploaded {} to s3://{}/{}.".format(file_name, bucket, object_name)
     LogEntry.objects.log_action(
         user_id=system_user.pk,
-        content_type_id=content_type.pk,
+        content_type_id=get_content_type().pk,
         object_id=None,
         object_repr="Executed database backup S3 upload",
         action_flag=ADDITION,
@@ -208,7 +211,7 @@ def download_file(bucket,
     msg = "Successfully downloaded s3 file {}/{} to {}.".format(bucket, object_name, file_name)
     LogEntry.objects.log_action(
         user_id=system_user.pk,
-        content_type_id=content_type.pk,
+        content_type_id=get_content_type().pk,
         object_id=None,
         object_repr="Executed database backup S3 download",
         action_flag=ADDITION,
@@ -293,7 +296,7 @@ def main(argv, sys_values, system_user):
     if arg_to_backup:
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Begining Database Backup",
             action_flag=ADDITION,
@@ -316,7 +319,7 @@ def main(argv, sys_values, system_user):
 
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Finished Database Backup",
             action_flag=ADDITION,
@@ -329,7 +332,7 @@ def main(argv, sys_values, system_user):
     elif arg_to_restore:
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Begining Database Restore",
             action_flag=ADDITION,
@@ -352,7 +355,7 @@ def main(argv, sys_values, system_user):
 
         LogEntry.objects.log_action(
             user_id=system_user.pk,
-            content_type_id=content_type.pk,
+            content_type_id=get_content_type().pk,
             object_id=None,
             object_repr="Finished Database Restore",
             action_flag=ADDITION,
@@ -377,7 +380,7 @@ def run_backup(arg):
             logger.error(f"Caught Exception in run_backup. Exception: {e}.")
             LogEntry.objects.log_action(
                 user_id=system_user.pk,
-                content_type_id=content_type.pk,
+                content_type_id=get_content_type().pk,
                 object_id=None,
                 object_repr="Exception in run_backup",
                 action_flag=ADDITION,

tdrs-backend/tdpservice/scheduling/test/test_db_backup.py

@@ -0,0 +1,69 @@
+"""Test cases for db_backup.py functions."""
+
+import os
+import pytest
+from tdpservice.scheduling.management import db_backup
+from django.contrib.admin.models import LogEntry
+
+@pytest.mark.django_db
+def test_backup_database(system_user):
+    """Test backup functionality."""
+    file_name = "/tmp/test_backup.pg"
+    ret = db_backup.backup_database(file_name, "",
+                                    "postgres://tdpuser:something_secure@postgres:5432/tdrs_test",
+                                    system_user)
+
+    assert ret is True
+    assert os.path.getsize(file_name) > 0
+    os.remove(file_name)
+    assert os.path.exists(file_name) is False
+
+@pytest.mark.django_db
+def test_backup_database_fail_on_backup(system_user):
+    """Test backup fails on psql non-zero return code."""
+    with pytest.raises(Exception) as e:
+        file_name = "/tmp/test_backup.pg"
+        db_backup.backup_database(file_name, "asdfasdfassfd",
+                                  "postgres://tdpuser:something_secure@postgres:5432/tdrs_test",
+                                  system_user)
+
+    assert str(e.value) == "pg_dump command failed with a non zero exit code."
+    assert os.path.exists(file_name) is False
+
+@pytest.mark.django_db
+def test_backup_database_fail_on_general_exception():
+    """Test backup succeeds but raises exception on string user for log entry."""
+    with pytest.raises(Exception) as e:
+        file_name = "/tmp/test_backup.pg"
+        db_backup.backup_database(file_name, "",
+                                  "postgres://tdpuser:something_secure@postgres:5432/tdrs_test",
+                                  "system_user")
+
+    assert str(e.value) == "'str' object has no attribute 'pk'"
+    assert os.path.exists(file_name) is True
+    os.remove(file_name)
+    assert os.path.exists(file_name) is False
+
+
+@pytest.mark.django_db
+def test_get_database_credentials():
+    """Test get credentials."""
+    creds = db_backup.get_database_credentials("postgres://tdpuser:something_secure@postgres:5432/tdrs_test")
+    assert creds == ["tdpuser", "something_secure", "postgres", "5432", "tdrs_test"]
+
+@pytest.mark.django_db
+def test_main_backup(mocker, system_user):
+    """Test call the main function."""
+    mocker.patch(
+        'tdpservice.scheduling.management.db_backup.upload_file',
+        return_value=True
+    )
+    sys_vals = {"DATABASE_URI": "postgres://tdpuser:something_secure@postgres:5432",
+                "DATABASE_DB_NAME": "tdrs_test",
+                "POSTGRES_CLIENT_DIR": "",
+                "S3_BUCKET": "",
+                "S3_REGION": ""}
+
+    db_backup.main(['-b', '-f', '/tmp/test_backup.pg'], sys_values=sys_vals, system_user=system_user)
+    assert LogEntry.objects.get(change_message="Begining database backup.").pk is not None
+    assert LogEntry.objects.get(change_message="Finished database backup.").pk is not None