From 2753507c877d0141fc3e72ce1325758cfb72d32a Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 14 Aug 2024 14:01:16 -0400 Subject: [PATCH 01/36] initial commit - added filter --- tdrs-backend/tdpservice/data_files/admin.py | 40 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 1a049dad3b..a07cc43ace 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -60,7 +60,44 @@ def data_file_summary(self, obj): return format_html("{field}", field=f'{df.id}' + ":" + df.get_status(), url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/") - + + class by_submission_date(admin.SimpleListFilter): + """filter data files by month.""" + title = 'Submission Time' + parameter_name = 'Submission Day/Month/Year' + + def lookups(self, request, model_admin): + """Return a list of tuples.""" + return [ + ('y', 'Yesterday'), + ('0', 'Today'), + ('7', 'Past 7 days'), + ('t-m', 'This month'), + ('t-y', 'This year'), + ] + + def queryset(self, request, queryset): + """Return a queryset.""" + from datetime import datetime, timedelta, timezone + yesterday = datetime.now(tz=timezone.utc) - timedelta(days=1) + this_month = datetime.now(tz=timezone.utc).replace(day=1) + this_year = datetime.now(tz=timezone.utc).replace(month=1, day=1) + if self.value() == 'y': + query_set_ids = [df.id for df in queryset if df.submitted_at == yesterday] + return queryset.filter(id__in=query_set_ids) + elif self.value() in : + last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=self.value()) + query_set_ids = [df.id for df in queryset if df.submitted_at >= last_login__lte] + return queryset.filter(id__in=query_set_ids) + elif self.value() == 30: + query_set_ids = [df.id for df in queryset if df.submitted_at >= this_month] + return queryset.filter(id__in=query_set_ids) + elif self.value() == 365: + query_set_ids = [df.id for df in queryset if df.submitted_at >= this_year] + return queryset.filter(id__in=query_set_ids) + else: + return queryset + list_display = [ 'id', 'stt', @@ -78,6 +115,7 @@ def data_file_summary(self, obj): 'stt', 'user', 'year', + by_submission_date, 'version', 'summary__status', DataFileSummaryPrgTypeFilter From b79cbb1f8ff3fa8ed7ae1e17e63b97bbdd91aee9 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 15 Aug 2024 09:42:34 -0400 Subject: [PATCH 02/36] 3076 added test --- tdrs-backend/tdpservice/data_files/admin.py | 28 +++++++++---------- .../tdpservice/data_files/test/test_admin.py | 27 ++++++++++++++++++ 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index a07cc43ace..73e520fdfd 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -63,37 +63,37 @@ def data_file_summary(self, obj): class by_submission_date(admin.SimpleListFilter): """filter data files by month.""" - title = 'Submission Time' + title = 'Submission Date' parameter_name = 'Submission Day/Month/Year' def lookups(self, request, model_admin): """Return a list of tuples.""" return [ - ('y', 'Yesterday'), + ('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), - ('t-m', 'This month'), - ('t-y', 'This year'), + ('30', 'This month'), + ('365', 'This year'), ] def queryset(self, request, queryset): """Return a queryset.""" from datetime import datetime, timedelta, timezone - yesterday = datetime.now(tz=timezone.utc) - timedelta(days=1) + yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) this_month = datetime.now(tz=timezone.utc).replace(day=1) this_year = datetime.now(tz=timezone.utc).replace(month=1, day=1) - if self.value() == 'y': - query_set_ids = [df.id for df in queryset if df.submitted_at == yesterday] + if self.value() == '1': + query_set_ids = [df.id for df in queryset if df.created_at.replace(hour=0, minute=0, second=0, microsecond=0) == yesterday] return queryset.filter(id__in=query_set_ids) - elif self.value() in : - last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=self.value()) - query_set_ids = [df.id for df in queryset if df.submitted_at >= last_login__lte] + elif self.value() in ['0', '7']: + last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) + query_set_ids = [df.id for df in queryset if df.created_at >= last_login__lte] return queryset.filter(id__in=query_set_ids) - elif self.value() == 30: - query_set_ids = [df.id for df in queryset if df.submitted_at >= this_month] + elif self.value() == '30': + query_set_ids = [df.id for df in queryset if df.created_at >= this_month] return queryset.filter(id__in=query_set_ids) - elif self.value() == 365: - query_set_ids = [df.id for df in queryset if df.submitted_at >= this_year] + elif self.value() == '365': + query_set_ids = [df.id for df in queryset if df.created_at >= this_year] return queryset.filter(id__in=query_set_ids) else: return queryset diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 02701fe825..5a1e695387 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -1,6 +1,7 @@ """Test DataFileAdmin methods.""" import pytest from django.contrib.admin.sites import AdminSite +#from django.contrib.admin.models import Admin from tdpservice.data_files.admin import DataFileAdmin from tdpservice.data_files.models import DataFile @@ -20,3 +21,29 @@ def test_DataFileAdmin_status(): DOMAIN = settings.FRONTEND_BASE_URL assert data_file_admin.error_report_link(data_file) == \ f"Parser Errors: 0" + +@pytest.mark.django_db +def test_by_submission_date(client): + """Test by_submission_date method.""" + from django.db.models.query import QuerySet + client.login(username='admin', password='password') + url= '/admin/parsers/datafile/' + # create fake queryset + fake_query = DataFile.objects.all() + data_file_admin = DataFileAdmin(DataFile, AdminSite()) + filter = DataFileAdmin.by_submission_date(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) + from django.contrib.admin.options import ModelAdmin + assert data_file_admin.by_submission_date.title == 'Submission Date' + assert data_file_admin.by_submission_date.parameter_name == 'Submission Day/Month/Year' + assert data_file_admin.by_submission_date.lookups (filter, None, None) == [('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year')] + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == False + df = DataFileFactory() + from datetime import datetime, timezone, timedelta + df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) + df.save() + fake_query = DataFile.objects.all() + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == True + df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=2) + df.save() + fake_query = DataFile.objects.all() + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == False From 1ca33a685b165a6dd9fa11bc6b0fd3616798c636 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 23 Aug 2024 09:57:13 -0400 Subject: [PATCH 03/36] linting --- tdrs-backend/tdpservice/data_files/admin.py | 17 +++++++++++------ .../tdpservice/data_files/test/test_admin.py | 14 ++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 73e520fdfd..e5cc94b743 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -60,9 +60,10 @@ def data_file_summary(self, obj): return format_html("{field}", field=f'{df.id}' + ":" + df.get_status(), url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/") - + class by_submission_date(admin.SimpleListFilter): """filter data files by month.""" + title = 'Submission Date' parameter_name = 'Submission Day/Month/Year' @@ -75,18 +76,22 @@ def lookups(self, request, model_admin): ('30', 'This month'), ('365', 'This year'), ] - + def queryset(self, request, queryset): """Return a queryset.""" from datetime import datetime, timedelta, timezone - yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) + yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) this_month = datetime.now(tz=timezone.utc).replace(day=1) this_year = datetime.now(tz=timezone.utc).replace(month=1, day=1) if self.value() == '1': - query_set_ids = [df.id for df in queryset if df.created_at.replace(hour=0, minute=0, second=0, microsecond=0) == yesterday] + query_set_ids = [df.id for df in queryset if df.created_at.replace( + hour=0, minute=0, second=0, microsecond=0 + ) == yesterday] return queryset.filter(id__in=query_set_ids) elif self.value() in ['0', '7']: - last_login__lte=datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) + last_login__lte = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) query_set_ids = [df.id for df in queryset if df.created_at >= last_login__lte] return queryset.filter(id__in=query_set_ids) elif self.value() == '30': @@ -97,7 +102,7 @@ def queryset(self, request, queryset): return queryset.filter(id__in=query_set_ids) else: return queryset - + list_display = [ 'id', 'stt', diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 5a1e695387..9239fe589e 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -1,7 +1,6 @@ """Test DataFileAdmin methods.""" import pytest from django.contrib.admin.sites import AdminSite -#from django.contrib.admin.models import Admin from tdpservice.data_files.admin import DataFileAdmin from tdpservice.data_files.models import DataFile @@ -25,25 +24,24 @@ def test_DataFileAdmin_status(): @pytest.mark.django_db def test_by_submission_date(client): """Test by_submission_date method.""" - from django.db.models.query import QuerySet client.login(username='admin', password='password') - url= '/admin/parsers/datafile/' # create fake queryset fake_query = DataFile.objects.all() data_file_admin = DataFileAdmin(DataFile, AdminSite()) filter = DataFileAdmin.by_submission_date(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) - from django.contrib.admin.options import ModelAdmin assert data_file_admin.by_submission_date.title == 'Submission Date' assert data_file_admin.by_submission_date.parameter_name == 'Submission Day/Month/Year' - assert data_file_admin.by_submission_date.lookups (filter, None, None) == [('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year')] - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == False + assert data_file_admin.by_submission_date.lookups(filter, None, None) == [ + ('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year') + ] + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is False df = DataFileFactory() from datetime import datetime, timezone, timedelta df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) df.save() fake_query = DataFile.objects.all() - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == True + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is True df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=2) df.save() fake_query = DataFile.objects.all() - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() == False + assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is False From 4a3a169ce71f75d8e070c7690d23f6e276b460cb Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 23 Aug 2024 10:51:28 -0400 Subject: [PATCH 04/36] move import to top --- tdrs-backend/tdpservice/data_files/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index e5cc94b743..46c4b5fc5f 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -6,6 +6,7 @@ from tdpservice.parsers.models import DataFileSummary, ParserError from django.conf import settings from django.utils.html import format_html +from datetime import datetime, timedelta, timezone DOMAIN = settings.FRONTEND_BASE_URL @@ -79,7 +80,7 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): """Return a queryset.""" - from datetime import datetime, timedelta, timezone + yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) From f810633a7ab48fc71b878a8b522a9e8e576c75c9 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 23 Aug 2024 10:52:04 -0400 Subject: [PATCH 05/36] lint --- tdrs-backend/tdpservice/data_files/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 46c4b5fc5f..d8ae51b5b9 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -80,7 +80,6 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): """Return a queryset.""" - yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) From bc050cf18e873d32822a3582e81c570a1f3ae3ea Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Tue, 27 Aug 2024 09:46:10 -0400 Subject: [PATCH 06/36] 3076 correction --- tdrs-backend/tdpservice/data_files/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index d8ae51b5b9..81d0401d2d 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -91,8 +91,8 @@ def queryset(self, request, queryset): ) == yesterday] return queryset.filter(id__in=query_set_ids) elif self.value() in ['0', '7']: - last_login__lte = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) - query_set_ids = [df.id for df in queryset if df.created_at >= last_login__lte] + last_week = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) + query_set_ids = [df.id for df in queryset if df.created_at >= last_week] return queryset.filter(id__in=query_set_ids) elif self.value() == '30': query_set_ids = [df.id for df in queryset if df.created_at >= this_month] From 16bfbceb4c55e8c45160d482b9f4a2f6559babcd Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 28 Aug 2024 04:02:26 -0400 Subject: [PATCH 07/36] 3076 Fixed today filter --- tdrs-backend/tdpservice/data_files/admin.py | 15 +++++---- .../tdpservice/data_files/test/test_admin.py | 31 +++++++++++++------ 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 81d0401d2d..03a065f65d 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -62,7 +62,7 @@ def data_file_summary(self, obj): field=f'{df.id}' + ":" + df.get_status(), url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/") - class by_submission_date(admin.SimpleListFilter): + class SubmissionDateFilter(admin.SimpleListFilter): """filter data files by month.""" title = 'Submission Date' @@ -83,16 +83,19 @@ def queryset(self, request, queryset): yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) - this_month = datetime.now(tz=timezone.utc).replace(day=1) - this_year = datetime.now(tz=timezone.utc).replace(month=1, day=1) + this_month = datetime.now(tz=timezone.utc).replace( + day=1, hour=0, minute=0, second=0, microsecond=0) + this_year = datetime.now(tz=timezone.utc).replace( + month=1, day=1, hour=0, minute=0, second=0, microsecond=0) + last_week_today = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) + last_week_today = last_week_today.replace(hour=0, minute=0, second=0, microsecond=0) if self.value() == '1': query_set_ids = [df.id for df in queryset if df.created_at.replace( hour=0, minute=0, second=0, microsecond=0 ) == yesterday] return queryset.filter(id__in=query_set_ids) elif self.value() in ['0', '7']: - last_week = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) - query_set_ids = [df.id for df in queryset if df.created_at >= last_week] + query_set_ids = [df.id for df in queryset if df.created_at >= last_week_today] return queryset.filter(id__in=query_set_ids) elif self.value() == '30': query_set_ids = [df.id for df in queryset if df.created_at >= this_month] @@ -120,7 +123,7 @@ def queryset(self, request, queryset): 'stt', 'user', 'year', - by_submission_date, + SubmissionDateFilter, 'version', 'summary__status', DataFileSummaryPrgTypeFilter diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 9239fe589e..3ddc9799d4 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -1,6 +1,7 @@ """Test DataFileAdmin methods.""" import pytest from django.contrib.admin.sites import AdminSite +from datetime import datetime, timezone, timedelta from tdpservice.data_files.admin import DataFileAdmin from tdpservice.data_files.models import DataFile @@ -22,26 +23,36 @@ def test_DataFileAdmin_status(): f"Parser Errors: 0" @pytest.mark.django_db -def test_by_submission_date(client): - """Test by_submission_date method.""" +def test_SubmissionDateFilter(client): + """Test SubmissionDateFilter method.""" client.login(username='admin', password='password') # create fake queryset fake_query = DataFile.objects.all() data_file_admin = DataFileAdmin(DataFile, AdminSite()) - filter = DataFileAdmin.by_submission_date(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) - assert data_file_admin.by_submission_date.title == 'Submission Date' - assert data_file_admin.by_submission_date.parameter_name == 'Submission Day/Month/Year' - assert data_file_admin.by_submission_date.lookups(filter, None, None) == [ + filter = DataFileAdmin.SubmissionDateFilter(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) + assert data_file_admin.SubmissionDateFilter.title == 'Submission Date' + assert data_file_admin.SubmissionDateFilter.parameter_name == 'Submission Day/Month/Year' + assert data_file_admin.SubmissionDateFilter.lookups(filter, None, None) == [ ('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year') ] - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is False + assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False df = DataFileFactory() - from datetime import datetime, timezone, timedelta df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) df.save() fake_query = DataFile.objects.all() - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is True + assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is True df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=2) df.save() fake_query = DataFile.objects.all() - assert data_file_admin.by_submission_date.queryset(filter, None, fake_query).exists() is False + assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False + + filter = DataFileAdmin.SubmissionDateFilter(None, {'Submission Day/Month/Year': '0'}, DataFile, DataFileAdmin) + df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) + df.save() + fake_query = DataFile.objects.all() + assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False + + df.created_at = datetime.now(tz=timezone.utc) + df.save() + fake_query = DataFile.objects.all() + assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is True From 3fa6d7aaea0d1bfab6172504482b1f7d455225c0 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 28 Aug 2024 04:18:11 -0400 Subject: [PATCH 08/36] linting --- tdrs-backend/tdpservice/data_files/admin.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 03a065f65d..54c943df94 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -80,27 +80,27 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): """Return a queryset.""" - yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( - hour=0, minute=0, second=0, microsecond=0 - ) - this_month = datetime.now(tz=timezone.utc).replace( - day=1, hour=0, minute=0, second=0, microsecond=0) - this_year = datetime.now(tz=timezone.utc).replace( - month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - last_week_today = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) - last_week_today = last_week_today.replace(hour=0, minute=0, second=0, microsecond=0) if self.value() == '1': + yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) query_set_ids = [df.id for df in queryset if df.created_at.replace( hour=0, minute=0, second=0, microsecond=0 ) == yesterday] return queryset.filter(id__in=query_set_ids) elif self.value() in ['0', '7']: + last_week_today = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) + last_week_today = last_week_today.replace(hour=0, minute=0, second=0, microsecond=0) query_set_ids = [df.id for df in queryset if df.created_at >= last_week_today] return queryset.filter(id__in=query_set_ids) elif self.value() == '30': + this_month = datetime.now(tz=timezone.utc).replace( + day=1, hour=0, minute=0, second=0, microsecond=0) query_set_ids = [df.id for df in queryset if df.created_at >= this_month] return queryset.filter(id__in=query_set_ids) elif self.value() == '365': + this_year = datetime.now(tz=timezone.utc).replace( + month=1, day=1, hour=0, minute=0, second=0, microsecond=0) query_set_ids = [df.id for df in queryset if df.created_at >= this_year] return queryset.filter(id__in=query_set_ids) else: From fb6ecb78c9893640ab7b2c8b3a6f9f2fd985b6c2 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 30 Aug 2024 10:34:16 -0400 Subject: [PATCH 09/36] 3076 applied a11y review changes --- tdrs-backend/tdpservice/data_files/admin/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 7043ffeed8..c0dfc2c141 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -57,14 +57,14 @@ def data_file_summary(self, obj): class SubmissionDateFilter(admin.SimpleListFilter): """filter data files by month.""" - title = 'Submission Date' + title = 'submission date' parameter_name = 'Submission Day/Month/Year' def lookups(self, request, model_admin): """Return a list of tuples.""" return [ - ('1', 'Yesterday'), ('0', 'Today'), + ('1', 'Yesterday'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year'), From 198b98ad04108917ad11da18b33b7a827d3bc723 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 30 Aug 2024 11:08:19 -0400 Subject: [PATCH 10/36] 3076 fix failing test --- tdrs-backend/tdpservice/data_files/test/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 8839483741..bbb9ae041d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -30,10 +30,10 @@ def test_SubmissionDateFilter(client): fake_query = DataFile.objects.all() data_file_admin = DataFileAdmin(DataFile, AdminSite()) filter = DataFileAdmin.SubmissionDateFilter(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) - assert data_file_admin.SubmissionDateFilter.title == 'Submission Date' + assert data_file_admin.SubmissionDateFilter.title == 'submission date' assert data_file_admin.SubmissionDateFilter.parameter_name == 'Submission Day/Month/Year' assert data_file_admin.SubmissionDateFilter.lookups(filter, None, None) == [ - ('1', 'Yesterday'), ('0', 'Today'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year') + ('0', 'Today'), ('1', 'Yesterday'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year') ] assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False df = DataFileFactory() From c61b298966cf7ef20bd4698f28dae429b37dddbc Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Tue, 3 Sep 2024 07:12:39 -0400 Subject: [PATCH 11/36] 3106 first draft --- tdrs-backend/tdpservice/data_files/admin/admin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index c0dfc2c141..9cb8dede64 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -23,9 +23,19 @@ def has_change_permission(self, request, obj=None): return False +@admin.action(description="Reparse selected data files") +def make_published(modeladmin, request, queryset): + """Reparse the selected data files.""" + import logging + logger = logging.getLogger(__name__) + logger.info(f"--------------Reparse selected data files: {queryset}") + for data_file in queryset: + data_file.reparse() + @admin.register(DataFile) class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" + actions = [make_published] def status(self, obj): """Return the status of the data file summary.""" From d59af030d3b7212cf54a32fd0132cf29595fc1f8 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 5 Sep 2024 15:05:35 -0400 Subject: [PATCH 12/36] clean up --- .../tdpservice/data_files/admin/admin.py | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 9cb8dede64..106f5a2115 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -64,50 +64,6 @@ def data_file_summary(self, obj): field=f'{df.id}' + ":" + df.get_status(), url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/") - class SubmissionDateFilter(admin.SimpleListFilter): - """filter data files by month.""" - - title = 'submission date' - parameter_name = 'Submission Day/Month/Year' - - def lookups(self, request, model_admin): - """Return a list of tuples.""" - return [ - ('0', 'Today'), - ('1', 'Yesterday'), - ('7', 'Past 7 days'), - ('30', 'This month'), - ('365', 'This year'), - ] - - def queryset(self, request, queryset): - """Return a queryset.""" - if self.value() == '1': - yesterday = (datetime.now(tz=timezone.utc) - timedelta(days=1)).replace( - hour=0, minute=0, second=0, microsecond=0 - ) - query_set_ids = [df.id for df in queryset if df.created_at.replace( - hour=0, minute=0, second=0, microsecond=0 - ) == yesterday] - return queryset.filter(id__in=query_set_ids) - elif self.value() in ['0', '7']: - last_week_today = datetime.now(tz=timezone.utc) - timedelta(days=int(self.value())) - last_week_today = last_week_today.replace(hour=0, minute=0, second=0, microsecond=0) - query_set_ids = [df.id for df in queryset if df.created_at >= last_week_today] - return queryset.filter(id__in=query_set_ids) - elif self.value() == '30': - this_month = datetime.now(tz=timezone.utc).replace( - day=1, hour=0, minute=0, second=0, microsecond=0) - query_set_ids = [df.id for df in queryset if df.created_at >= this_month] - return queryset.filter(id__in=query_set_ids) - elif self.value() == '365': - this_year = datetime.now(tz=timezone.utc).replace( - month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - query_set_ids = [df.id for df in queryset if df.created_at >= this_year] - return queryset.filter(id__in=query_set_ids) - else: - return queryset - inlines = [DataFileInline] list_display = [ From e1a3bbc86beca90b5f6831958704e868ce4cd407 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 5 Sep 2024 15:06:38 -0400 Subject: [PATCH 13/36] clean up --- .../tdpservice/data_files/test/test_admin.py | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index bbb9ae041d..2a2a352bb6 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -21,38 +21,3 @@ def test_DataFileAdmin_status(): DOMAIN = settings.FRONTEND_BASE_URL assert data_file_admin.error_report_link(data_file) == \ f"Parser Errors: 0" - -@pytest.mark.django_db -def test_SubmissionDateFilter(client): - """Test SubmissionDateFilter method.""" - client.login(username='admin', password='password') - # create fake queryset - fake_query = DataFile.objects.all() - data_file_admin = DataFileAdmin(DataFile, AdminSite()) - filter = DataFileAdmin.SubmissionDateFilter(None, {'Submission Day/Month/Year': '1'}, DataFile, DataFileAdmin) - assert data_file_admin.SubmissionDateFilter.title == 'submission date' - assert data_file_admin.SubmissionDateFilter.parameter_name == 'Submission Day/Month/Year' - assert data_file_admin.SubmissionDateFilter.lookups(filter, None, None) == [ - ('0', 'Today'), ('1', 'Yesterday'), ('7', 'Past 7 days'), ('30', 'This month'), ('365', 'This year') - ] - assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False - df = DataFileFactory() - df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) - df.save() - fake_query = DataFile.objects.all() - assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is True - df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=2) - df.save() - fake_query = DataFile.objects.all() - assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False - - filter = DataFileAdmin.SubmissionDateFilter(None, {'Submission Day/Month/Year': '0'}, DataFile, DataFileAdmin) - df.created_at = datetime.now(tz=timezone.utc) - timedelta(days=1) - df.save() - fake_query = DataFile.objects.all() - assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is False - - df.created_at = datetime.now(tz=timezone.utc) - df.save() - fake_query = DataFile.objects.all() - assert data_file_admin.SubmissionDateFilter.queryset(filter, None, fake_query).exists() is True From d3fd407cd23a0a69af2020bba1da1d245ae42fb1 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 5 Sep 2024 17:05:18 -0400 Subject: [PATCH 14/36] clean admin --- tdrs-backend/tdpservice/data_files/admin/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 106f5a2115..da125d5a6f 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -83,7 +83,6 @@ def data_file_summary(self, obj): 'stt', 'user', 'year', - SubmissionDateFilter, 'version', 'summary__status', DataFileSummaryPrgTypeFilter, From 223768685ff0dd364025bb4e2337fa95f0ef0b52 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 5 Sep 2024 17:05:26 -0400 Subject: [PATCH 15/36] clean admin --- .../search_indexes/management/commands/clean_and_reparse.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index a3b746a666..9ab56ac1d2 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -32,6 +32,7 @@ def add_arguments(self, parser): parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.") parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, " "fiscal_year/quarter aren't necessary.") + parser.add_argument("-f", "--files", nargs='+', type=int, help="Reparse specific datafiles by ID.") def __get_log_context(self, system_user): """Return logger context.""" @@ -228,8 +229,11 @@ def handle(self, *args, **options): fiscal_year = options.get('fiscal_year', None) fiscal_quarter = options.get('fiscal_quarter', None) reparse_all = options.get('all', False) + selected_files = options.get('files', None) new_indices = reparse_all is True + print('_______ selected_files:', selected_files) + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all if not args_passed: From 778f706e562acdc5ce2615610d06150f0f44537c Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Mon, 9 Sep 2024 10:08:48 -0400 Subject: [PATCH 16/36] parsing command --- .../tdpservice/data_files/admin/admin.py | 13 +++---- .../management/commands/clean_and_reparse.py | 34 +++++++++++++------ 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index da125d5a6f..f5de13b6da 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -7,6 +7,8 @@ from django.conf import settings from django.utils.html import format_html from datetime import datetime, timedelta, timezone +from django.core.management import call_command +from django.core.management.base import CommandError DOMAIN = settings.FRONTEND_BASE_URL @@ -24,18 +26,17 @@ def has_change_permission(self, request, obj=None): @admin.action(description="Reparse selected data files") -def make_published(modeladmin, request, queryset): +def reparse_cmd(modeladmin, request, queryset): """Reparse the selected data files.""" import logging - logger = logging.getLogger(__name__) - logger.info(f"--------------Reparse selected data files: {queryset}") - for data_file in queryset: - data_file.reparse() + # Reparse the selected data files using management command + files=queryset.values_list("id", flat=True) + call_command("clean_and_reparse", f'-f {",".join(map(str, files))}') @admin.register(DataFile) class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" - actions = [make_published] + actions = [reparse_cmd] def status(self, obj): """Return the status of the data file summary.""" diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index 9ab56ac1d2..4bcd9e4704 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -16,6 +16,7 @@ from django.utils import timezone from django.conf import settings import logging +from django.core import serializers logger = logging.getLogger(__name__) @@ -32,7 +33,12 @@ def add_arguments(self, parser): parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.") parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, " "fiscal_year/quarter aren't necessary.") - parser.add_argument("-f", "--files", nargs='+', type=int, help="Reparse specific datafiles by ID.") + + # we can use "from django.core import serializers" to serialize the datafiles and then use the datafiles to reparse + # the problem is that we need to serialize the datafiles before we can use them + # if the number of datafiles is small, we can serialize all of them and then use them to reparse + # if the number of datafiles is large, we can serialize them in batches and then use them to reparse + parser.add_argument("-f", "--files", nargs='+', type=str, help="Re-parse specific datafiles by datafile id") def __get_log_context(self, system_user): """Return logger context.""" @@ -230,11 +236,11 @@ def handle(self, *args, **options): fiscal_quarter = options.get('fiscal_quarter', None) reparse_all = options.get('all', False) selected_files = options.get('files', None) + selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None new_indices = reparse_all is True - print('_______ selected_files:', selected_files) - - args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all + # have to check if the selected_files is not None + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files if not args_passed: logger.warn("No arguments supplied.") @@ -247,8 +253,8 @@ def handle(self, *args, **options): if reparse_all: backup_file_name += "_FY_All_Q1-4" continue_msg = continue_msg.format(fy="All", q="Q1-4") - else: - if not fiscal_year and not fiscal_quarter: + else: # check for selected_files + if not fiscal_year and not fiscal_quarter and not selected_files: print( 'Options --fiscal_year and --fiscal_quarter not set. ' 'Provide either option to continue, or --all to wipe all submissions.' @@ -266,18 +272,24 @@ def handle(self, *args, **options): files = files.filter(quarter=fiscal_quarter) backup_file_name += f"_FY_All_{fiscal_quarter}" continue_msg = continue_msg.format(fy="All", q=fiscal_quarter) + elif selected_files: + files = files.filter(id__in=selected_files) + backup_file_name += f"_selected_files" + continue_msg = continue_msg.format(fy="All", q="Q1-4") + fmt_str = "be" if new_indices else "NOT be" continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str) - + num_files = files.count() fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})" continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str) - c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower() - if c not in ['y', 'yes']: - print('Cancelled.') - return + if not selected_files: + c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower() + if c not in ['y', 'yes']: + print('Cancelled.') + return system_user, created = User.objects.get_or_create(username='system') if created: From a5737b4a09f944ed8fb16ca123af7cb02e2caf16 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Mon, 9 Sep 2024 10:09:57 -0400 Subject: [PATCH 17/36] delete unwanted imports --- tdrs-backend/tdpservice/data_files/admin/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index f5de13b6da..38d34272f6 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -6,7 +6,6 @@ from tdpservice.data_files.admin.filters import DataFileSummaryPrgTypeFilter, LatestReparseEvent from django.conf import settings from django.utils.html import format_html -from datetime import datetime, timedelta, timezone from django.core.management import call_command from django.core.management.base import CommandError From 7fa47a8e74138177fe1a41ad3b0667ba15e5a659 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Tue, 10 Sep 2024 20:31:45 -0400 Subject: [PATCH 18/36] added modal form --- .../tdpservice/data_files/admin/admin.py | 44 ++++++++++++++----- .../tdpservice/data_files/test/test_admin.py | 1 - .../management/commands/clean_and_reparse.py | 19 +++----- .../templates/admin/action_confirmation.html | 11 +++++ 4 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 tdrs-backend/tdpservice/templates/admin/action_confirmation.html diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 38d34272f6..48bc61a291 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -7,7 +7,6 @@ from django.conf import settings from django.utils.html import format_html from django.core.management import call_command -from django.core.management.base import CommandError DOMAIN = settings.FRONTEND_BASE_URL @@ -23,19 +22,42 @@ def has_change_permission(self, request, obj=None): """Read only permissions.""" return False - -@admin.action(description="Reparse selected data files") -def reparse_cmd(modeladmin, request, queryset): - """Reparse the selected data files.""" - import logging - # Reparse the selected data files using management command - files=queryset.values_list("id", flat=True) - call_command("clean_and_reparse", f'-f {",".join(map(str, files))}') - @admin.register(DataFile) class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" - actions = [reparse_cmd] + + actions = ['reparse_cmd'] + + #@admin.action(description="Reparse selected data files") + def reparse_cmd(self, request, queryset): + """Reparse the selected data files.""" + if request.POST.get('post'): + files = queryset.values_list("id", flat=True) + self.message_user(request, f"Reparse command has been sent to the Celery queue for {len(files)} data files.") + from django.http import HttpResponseRedirect + return None + #return HttpResponseRedirect(request.get_full_path()) + #call_command("clean_and_reparse", f'-f {",".join(map(str, files))}') + #modeladmin.message_user(request, "Data files reparse command has been sent to the Celery queue." + self.message_user(request, "Data files reparse command has been sent to the Celery queue.") + from django.shortcuts import redirect + from django.urls import reverse + url = reverse('admin:search_indexes_reparsemeta_changelist') + return redirect(url) + else: + request.current_app = self.admin_site.name + from django.template.response import TemplateResponse + return TemplateResponse(request, "admin/action_confirmation.html") + + # TODO: add tests for this method + def get_actions(self, request): + actions = super().get_actions(request) + if not request.user.groups.filter( + name__in=["OFA System Admin", "OFA Admin"] + ).exists(): + if "reparse_cmd" in actions: + del actions["reparse_cmd"] + return actions def status(self, obj): """Return the status of the data file summary.""" diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 2a2a352bb6..c11b1bd6fa 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -1,7 +1,6 @@ """Test DataFileAdmin methods.""" import pytest from django.contrib.admin.sites import AdminSite -from datetime import datetime, timezone, timedelta from tdpservice.data_files.admin.admin import DataFileAdmin from tdpservice.data_files.models import DataFile diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index 4bcd9e4704..e5dbc8e6ed 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -16,7 +16,6 @@ from django.utils import timezone from django.conf import settings import logging -from django.core import serializers logger = logging.getLogger(__name__) @@ -33,11 +32,6 @@ def add_arguments(self, parser): parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.") parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, " "fiscal_year/quarter aren't necessary.") - - # we can use "from django.core import serializers" to serialize the datafiles and then use the datafiles to reparse - # the problem is that we need to serialize the datafiles before we can use them - # if the number of datafiles is small, we can serialize all of them and then use them to reparse - # if the number of datafiles is large, we can serialize them in batches and then use them to reparse parser.add_argument("-f", "--files", nargs='+', type=str, help="Re-parse specific datafiles by datafile id") def __get_log_context(self, system_user): @@ -250,10 +244,14 @@ def handle(self, *args, **options): backup_file_name = "/tmp/reparsing_backup" files = DataFile.objects.all() continue_msg = "You have selected to reparse datafiles for FY {fy} and {q}. The reparsed files " + if selected_files: + files = files.filter(id__in=selected_files) + backup_file_name += "_selected_files" + continue_msg = continue_msg.format(fy="All", q="Q1-4") if reparse_all: backup_file_name += "_FY_All_Q1-4" continue_msg = continue_msg.format(fy="All", q="Q1-4") - else: # check for selected_files + else: if not fiscal_year and not fiscal_quarter and not selected_files: print( 'Options --fiscal_year and --fiscal_quarter not set. ' @@ -272,15 +270,10 @@ def handle(self, *args, **options): files = files.filter(quarter=fiscal_quarter) backup_file_name += f"_FY_All_{fiscal_quarter}" continue_msg = continue_msg.format(fy="All", q=fiscal_quarter) - elif selected_files: - files = files.filter(id__in=selected_files) - backup_file_name += f"_selected_files" - continue_msg = continue_msg.format(fy="All", q="Q1-4") - fmt_str = "be" if new_indices else "NOT be" continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str) - + num_files = files.count() fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})" continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str) diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html new file mode 100644 index 0000000000..6bf9365e3e --- /dev/null +++ b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html @@ -0,0 +1,11 @@ +{% load l10n %} +
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} + + + +
+
\ No newline at end of file From 1463ea4c1c70d1efaf362f41e665e7867efb93c4 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 11 Sep 2024 07:39:11 -0400 Subject: [PATCH 19/36] added styling to confirmation page --- .../tdpservice/data_files/admin/admin.py | 1 + .../templates/admin/action_confirmation.html | 105 ++++++++++++++++-- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 514928eadf..b8c71b8fce 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -49,6 +49,7 @@ def reparse_cmd(self, request, queryset): else: request.current_app = self.admin_site.name from django.template.response import TemplateResponse + # add information about reparsing here to this page return TemplateResponse(request, "admin/action_confirmation.html") # TODO: add tests for this method diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html index 6bf9365e3e..158ecea4aa 100644 --- a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html +++ b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html @@ -1,11 +1,96 @@ -{% load l10n %} -
{% csrf_token %} -
- {% for obj in queryset %} - - {% endfor %} - - - + + + + + + + + + + + + + + + Page not found - TANF Data Portal + + + + +
+ Skip to main content +
U.S. flag

An Official website of the United States government

+
+
+
+ +
+ +
+
+
+
+
+
+
+
+

Confirm

+ +
+
+
+
+
+
+
+
- \ No newline at end of file + + + From 152d43964f25194fc1225c9c11ac98ccbd5271f6 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 11 Sep 2024 11:55:11 -0400 Subject: [PATCH 20/36] changes --- .../tdpservice/data_files/admin/admin.py | 16 +++++++++++++++- tdrs-backend/tdpservice/data_files/views.py | 7 +++++++ .../templates/admin/action_confirmation.html | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index b8c71b8fce..fd57eb4d41 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -33,6 +33,7 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): #@admin.action(description="Reparse selected data files") def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" + # TOTO: remove this if part. This is just for testing if request.POST.get('post'): files = queryset.values_list("id", flat=True) self.message_user(request, f"Reparse command has been sent to the Celery queue for {len(files)} data files.") @@ -50,7 +51,20 @@ def reparse_cmd(self, request, queryset): request.current_app = self.admin_site.name from django.template.response import TemplateResponse # add information about reparsing here to this page - return TemplateResponse(request, "admin/action_confirmation.html") + number_of_files = queryset.count() + number_of_records = sum([df.summary.total_number_of_records_in_file for df in queryset]) + context = dict( + self.admin_site.each_context(request), + title="Are you sure?", + action="reparse_cmd", + queryset=queryset, + opts=self.model._meta, + msg = f"{number_of_files} datafiles, {number_of_records} records will be lost" + ) + # template should hit an action endpoint, which will call the command + # that endpoint will then redirect to the search_indexes_reparsemeta_changelist + + return TemplateResponse(request, "admin/action_confirmation.html", context) # TODO: add tests for this method def get_actions(self, request): diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 3f67d7cb3f..bbc5b27c43 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -150,6 +150,13 @@ def download_error_report(self, request, pk=None): parser_errors = ParserError.objects.all().filter(file=datafile) serializer = ParsingErrorSerializer(parser_errors, many=True, context=self.get_serializer_context()) return Response(get_xls_serialized_file(serializer.data)) + + @action(methods=["get"], detail=False) + def run_action_reparse_cmd(self, request, pk=None): + """Run the reparse command.""" + # Need to add the reparsing file ids to the request object + print('________ request: ', request.__dict__) + class GetYearList(APIView): diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html index 158ecea4aa..25269f1ec2 100644 --- a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html +++ b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html @@ -54,6 +54,7 @@

Confirm

+ {{ msg}}
From d3deddfa803111d7abbbdc2fb9fa69a0487a8493 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 13 Sep 2024 08:58:48 -0400 Subject: [PATCH 21/36] 3106 middle page works --- .../tdpservice/data_files/admin/admin.py | 7 ++++-- tdrs-backend/tdpservice/data_files/views.py | 12 ++++++--- .../templates/admin/action_confirmation.html | 25 +++++++++++++++++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index fd57eb4d41..60809eff4e 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -34,8 +34,10 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" # TOTO: remove this if part. This is just for testing + files = queryset.values_list("id", flat=True) + file_ids = ",".join(map(str, files)) + file_ids = f'[{file_ids}]' if request.POST.get('post'): - files = queryset.values_list("id", flat=True) self.message_user(request, f"Reparse command has been sent to the Celery queue for {len(files)} data files.") from django.http import HttpResponseRedirect return None @@ -59,7 +61,8 @@ def reparse_cmd(self, request, queryset): action="reparse_cmd", queryset=queryset, opts=self.model._meta, - msg = f"{number_of_files} datafiles, {number_of_records} records will be lost" + msg = f"{number_of_files} datafiles, {number_of_records} records will be lost", + file_ids=file_ids, ) # template should hit an action endpoint, which will call the command # that endpoint will then redirect to the search_indexes_reparsemeta_changelist diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index bbc5b27c43..bbf8291fdb 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -150,12 +150,18 @@ def download_error_report(self, request, pk=None): parser_errors = ParserError.objects.all().filter(file=datafile) serializer = ParsingErrorSerializer(parser_errors, many=True, context=self.get_serializer_context()) return Response(get_xls_serialized_file(serializer.data)) - - @action(methods=["get"], detail=False) + + + from rest_framework.parsers import JSONParser, MultiPartParser, FormParser + + @action(methods=["post"], detail=False, parser_classes=[JSONParser, MultiPartParser, FormParser]) def run_action_reparse_cmd(self, request, pk=None): """Run the reparse command.""" # Need to add the reparsing file ids to the request object - print('________ request: ', request.__dict__) + print('________ request: ', request.data) + from django.core.management import call_command + call_command("clean_and_reparse", f"-f {request.data['file_ids']}") + return Response({'status': 'success'}) diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html index 25269f1ec2..a23827de98 100644 --- a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html +++ b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html @@ -54,8 +54,8 @@

Confirm

- {{ msg}} - + {{ msg}}
{{ file_ids }} +
@@ -94,4 +94,25 @@

Confirm

+ + From a54f14ae952aa6a5257ddd5874db8d7addd93296 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 13 Sep 2024 10:29:29 -0400 Subject: [PATCH 22/36] 3106 can add files for reparsing --- .../tdpservice/data_files/admin/admin.py | 50 ++++++------------- tdrs-backend/tdpservice/data_files/views.py | 6 +-- .../templates/admin/action_confirmation.html | 9 ++-- 3 files changed, 21 insertions(+), 44 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index a8dfafbd6f..b8ff66b515 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -7,9 +7,8 @@ from tdpservice.data_files.admin.filters import DataFileSummaryPrgTypeFilter, LatestReparseEvent, VersionFilter from django.conf import settings from django.utils.html import format_html -from django.core.management import call_command -from django.core.management.base import CommandError from datetime import datetime, timedelta, timezone +from django.template.response import TemplateResponse DOMAIN = settings.FRONTEND_BASE_URL @@ -36,39 +35,20 @@ def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" # TOTO: remove this if part. This is just for testing files = queryset.values_list("id", flat=True) - file_ids = ",".join(map(str, files)) - file_ids = f'[{file_ids}]' - if request.POST.get('post'): - self.message_user(request, f"Reparse command has been sent to the Celery queue for {len(files)} data files.") - from django.http import HttpResponseRedirect - return None - #return HttpResponseRedirect(request.get_full_path()) - #call_command("clean_and_reparse", f'-f {",".join(map(str, files))}') - #modeladmin.message_user(request, "Data files reparse command has been sent to the Celery queue." - self.message_user(request, "Data files reparse command has been sent to the Celery queue.") - from django.shortcuts import redirect - from django.urls import reverse - url = reverse('admin:search_indexes_reparsemeta_changelist') - return redirect(url) - else: - request.current_app = self.admin_site.name - from django.template.response import TemplateResponse - # add information about reparsing here to this page - number_of_files = queryset.count() - number_of_records = sum([df.summary.total_number_of_records_in_file for df in queryset]) - context = dict( - self.admin_site.each_context(request), - title="Are you sure?", - action="reparse_cmd", - queryset=queryset, - opts=self.model._meta, - msg = f"{number_of_files} datafiles, {number_of_records} records will be lost", - file_ids=file_ids, - ) - # template should hit an action endpoint, which will call the command - # that endpoint will then redirect to the search_indexes_reparsemeta_changelist - - return TemplateResponse(request, "admin/action_confirmation.html", context) + file_ids = ",".join(map(str, files)) + #request.current_app = self.admin_site.name + number_of_files = queryset.count() + number_of_records = sum([df.summary.total_number_of_records_in_file for df in queryset]) + context = dict( + self.admin_site.each_context(request), + title="Are you sure?", + action="reparse_cmd", + queryset=queryset, + opts=self.model._meta, + msg = f"{number_of_files} datafiles, {number_of_records} records will be lost", + file_ids=file_ids, + ) + return TemplateResponse(request, "admin/action_confirmation.html", context) # TODO: add tests for this method def get_actions(self, request): diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index bbf8291fdb..2b85856ff0 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -22,6 +22,8 @@ from tdpservice.data_files.s3_client import S3Client from tdpservice.parsers.models import ParserError from tdpservice.parsers.serializers import ParsingErrorSerializer +from rest_framework.parsers import JSONParser, MultiPartParser, FormParser +from django.core.management import call_command logger = logging.getLogger(__name__) @@ -152,14 +154,10 @@ def download_error_report(self, request, pk=None): return Response(get_xls_serialized_file(serializer.data)) - from rest_framework.parsers import JSONParser, MultiPartParser, FormParser - @action(methods=["post"], detail=False, parser_classes=[JSONParser, MultiPartParser, FormParser]) def run_action_reparse_cmd(self, request, pk=None): """Run the reparse command.""" # Need to add the reparsing file ids to the request object - print('________ request: ', request.data) - from django.core.management import call_command call_command("clean_and_reparse", f"-f {request.data['file_ids']}") return Response({'status': 'success'}) diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html index a23827de98..affc76fb3f 100644 --- a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html +++ b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html @@ -53,9 +53,10 @@
-

Confirm

- {{ msg}}
{{ file_ids }} - +

Reparsing...

+
+ {{ msg}}

+
@@ -108,8 +109,6 @@

Confirm

formData = new FormData(); formData.append('file_ids', '{{ file_ids }}'); data = {'file_ids': '{{ file_ids }}'}; - console.log('____________' + data); - console.log('____________' + JSON.stringify(data)); console.log(xhr) xhr.send(JSON.stringify(data)); } From 2b75af88360d76bdeec5d5f6ae519be4a1dceb1a Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 13 Sep 2024 11:00:54 -0400 Subject: [PATCH 23/36] 3106 Correct linting --- tdrs-backend/tdpservice/data_files/admin/admin.py | 11 ++++------- tdrs-backend/tdpservice/data_files/test/test_admin.py | 1 + tdrs-backend/tdpservice/data_files/views.py | 4 +--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index b8ff66b515..0348981b36 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -30,13 +30,11 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): actions = ['reparse_cmd'] - #@admin.action(description="Reparse selected data files") def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" # TOTO: remove this if part. This is just for testing files = queryset.values_list("id", flat=True) - file_ids = ",".join(map(str, files)) - #request.current_app = self.admin_site.name + file_ids = ",".join(map(str, files)) number_of_files = queryset.count() number_of_records = sum([df.summary.total_number_of_records_in_file for df in queryset]) context = dict( @@ -45,17 +43,16 @@ def reparse_cmd(self, request, queryset): action="reparse_cmd", queryset=queryset, opts=self.model._meta, - msg = f"{number_of_files} datafiles, {number_of_records} records will be lost", + msg=f"{number_of_files} datafiles, {number_of_records} records will be lost", file_ids=file_ids, ) return TemplateResponse(request, "admin/action_confirmation.html", context) # TODO: add tests for this method def get_actions(self, request): + """Return the actions.""" actions = super().get_actions(request) - if not request.user.groups.filter( - name__in=["OFA System Admin", "OFA Admin"] - ).exists(): + if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists(): if "reparse_cmd" in actions: del actions["reparse_cmd"] return actions diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py index 25bd04430c..bbb9ae041d 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_admin.py +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -1,6 +1,7 @@ """Test DataFileAdmin methods.""" import pytest from django.contrib.admin.sites import AdminSite +from datetime import datetime, timezone, timedelta from tdpservice.data_files.admin.admin import DataFileAdmin from tdpservice.data_files.models import DataFile diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 2b85856ff0..caf40fdec4 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -22,7 +22,7 @@ from tdpservice.data_files.s3_client import S3Client from tdpservice.parsers.models import ParserError from tdpservice.parsers.serializers import ParsingErrorSerializer -from rest_framework.parsers import JSONParser, MultiPartParser, FormParser +from rest_framework.parsers import JSONParser, FormParser from django.core.management import call_command logger = logging.getLogger(__name__) @@ -153,7 +153,6 @@ def download_error_report(self, request, pk=None): serializer = ParsingErrorSerializer(parser_errors, many=True, context=self.get_serializer_context()) return Response(get_xls_serialized_file(serializer.data)) - @action(methods=["post"], detail=False, parser_classes=[JSONParser, MultiPartParser, FormParser]) def run_action_reparse_cmd(self, request, pk=None): """Run the reparse command.""" @@ -162,7 +161,6 @@ def run_action_reparse_cmd(self, request, pk=None): return Response({'status': 'success'}) - class GetYearList(APIView): """Get list of years for which there are data_files.""" From 626087f9dd68943884a3903b2378fb56ef05fc8b Mon Sep 17 00:00:00 2001 From: andrew-jameson Date: Tue, 17 Sep 2024 13:51:19 -0400 Subject: [PATCH 24/36] bypassing status check to get this deployed for testing --- .github/workflows/deploy-on-label.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 8d4720f07c..65dc199bb4 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -71,7 +71,7 @@ jobs: - name: Circle CI Deployment Trigger id: curl-circle-ci - if: steps.get-pr-state.outputs.STATE == 'success' + #if: steps.get-pr-state.outputs.STATE == 'success' uses: promiseofcake/circleci-trigger-action@v1 with: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} From 38d50f54eacec14cd0d4557387db82d2f7812db4 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 18 Sep 2024 09:59:25 -0400 Subject: [PATCH 25/36] injected working js --- .../tdpservice/data_files/admin/admin.py | 3 + .../static/admin/js/admin/mymodel.js | 36 ++++++ .../templates/admin/action_confirmation.html | 117 ------------------ .../tdpservice/templates/error_pages/500.html | 105 ---------------- 4 files changed, 39 insertions(+), 222 deletions(-) create mode 100644 tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js delete mode 100644 tdrs-backend/tdpservice/templates/admin/action_confirmation.html delete mode 100644 tdrs-backend/tdpservice/templates/error_pages/500.html diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 0348981b36..89d70477c5 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -28,6 +28,9 @@ def has_change_permission(self, request, obj=None): class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" + class Media: + js = ('admin/js/admin/mymodel.js',) + actions = ['reparse_cmd'] def reparse_cmd(self, request, queryset): diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js new file mode 100644 index 0000000000..ac945f0944 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js @@ -0,0 +1,36 @@ +// get current form by id +/* +$('button[type="submit"]').submit(function() { + //this.submit(); + console.log('submitting'); + disableFields(); // your own function + return false; +}); +*/ + +$(window).on('load', function() { + //your code here + console.log('loaded'); + var S=document.querySelector('button[type=submit]'); + console.log(S); + // add the first listener + var theForm = S.parentNode.parentNode; + console.log(theForm); + S.addEventListener('click', function(e) { + e.preventDefault(); + console.log('submitting'); + disableFields(); // your own function + e.preventDefault(); + + alert("Ensure you input a value in both fields!"); + theForm.submit(); //this is working + }); + + +}); + + +disableFields = function() { + console.log('disabling fields'); +} + diff --git a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html b/tdrs-backend/tdpservice/templates/admin/action_confirmation.html deleted file mode 100644 index affc76fb3f..0000000000 --- a/tdrs-backend/tdpservice/templates/admin/action_confirmation.html +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - Page not found - TANF Data Portal - - - - -
- Skip to main content -
U.S. flag

An Official website of the United States government

-
-
-
- -
- -
-
-
-
-
-
-
-
-

Reparsing...

-
- {{ msg}}

- -
-
-
-
-
-
-
- -
- - - - - diff --git a/tdrs-backend/tdpservice/templates/error_pages/500.html b/tdrs-backend/tdpservice/templates/error_pages/500.html deleted file mode 100644 index 0060aa69fa..0000000000 --- a/tdrs-backend/tdpservice/templates/error_pages/500.html +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - - - - - - - - - - Page not found - TANF Data Portal - - - - -
- Skip to main content -
U.S. flag

An Official website of the United States government

-
-
-
- -
- -
-
-
-
-
-
-
-
-

Error

-

We’re sorry, there was an error in response.

- -

{{ error }}.

-
- - - Contact Us - -
-
-
-
-
-
-
-
- -
- - - \ No newline at end of file From 5e6c89c4bb1bdb25ad551bdbfcad6535a49ffe46 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 18 Sep 2024 13:27:30 -0400 Subject: [PATCH 26/36] remove some of the comments --- .../tdpservice/data_files/admin/admin.py | 4 +- .../static/admin/js/admin/mymodel.js | 48 +++++++++++++++---- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 89d70477c5..5598c451b1 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -36,6 +36,8 @@ class Media: def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" # TOTO: remove this if part. This is just for testing + # we can run the command directly here + files = queryset.values_list("id", flat=True) file_ids = ",".join(map(str, files)) number_of_files = queryset.count() @@ -49,7 +51,7 @@ def reparse_cmd(self, request, queryset): msg=f"{number_of_files} datafiles, {number_of_records} records will be lost", file_ids=file_ids, ) - return TemplateResponse(request, "admin/action_confirmation.html", context) + #return TemplateResponse(request, "admin/action_confirmation.html", context) # TODO: add tests for this method def get_actions(self, request): diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js index ac945f0944..1253151396 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js @@ -1,13 +1,3 @@ -// get current form by id -/* -$('button[type="submit"]').submit(function() { - //this.submit(); - console.log('submitting'); - disableFields(); // your own function - return false; -}); -*/ - $(window).on('load', function() { //your code here console.log('loaded'); @@ -34,3 +24,41 @@ disableFields = function() { console.log('disabling fields'); } + +reparseFiles = function() { + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function () { + if (xhr.readyState === 4) { + alert(xhr.response); + } + } + xhr.open('POST', '/v1/data_files/run_action_reparse_cmd/', false); + xhr.setRequestHeader("X-CSRFToken", '{{ csrf_token }}'); + xhr.setRequestHeader("Content-Type", "application/json;charset=UTF-8"); + formData = new FormData(); + formData.append('file_ids', '{{ file_ids }}'); + data = {'file_ids': '{{ file_ids }}'}; + console.log(xhr) + xhr.send(JSON.stringify(data)); +} + +/* + +*/ From ed965ac36996caffb52b3066907794f3a72ad0e8 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 19 Sep 2024 07:41:05 -0400 Subject: [PATCH 27/36] revert changes --- .../management/commands/clean_and_reparse.py | 46 +++++--- .../tdpservice/templates/error_pages/500.html | 105 ++++++++++++++++++ 2 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 tdrs-backend/tdpservice/templates/error_pages/500.html diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index e5dbc8e6ed..03706ec3d7 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -224,23 +224,8 @@ def __calculate_timeout(self, num_files, num_records): logger.info(f"Setting timeout for the reparse event to be {delta} seconds from meta model creation date.") return delta - def handle(self, *args, **options): - """Delete and reparse datafiles matching a query.""" - fiscal_year = options.get('fiscal_year', None) - fiscal_quarter = options.get('fiscal_quarter', None) - reparse_all = options.get('all', False) - selected_files = options.get('files', None) - selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None - new_indices = reparse_all is True - - # have to check if the selected_files is not None - args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files - - if not args_passed: - logger.warn("No arguments supplied.") - self.print_help("manage.py", "clean_and_parse") - return - + def get_files_to_reparse(case, fiscal_year, fiscal_quarter, selected_files, reparse_all): + """Get the files to reparse.""" backup_file_name = "/tmp/reparsing_backup" files = DataFile.objects.all() continue_msg = "You have selected to reparse datafiles for FY {fy} and {q}. The reparsed files " @@ -270,6 +255,33 @@ def handle(self, *args, **options): files = files.filter(quarter=fiscal_quarter) backup_file_name += f"_FY_All_{fiscal_quarter}" continue_msg = continue_msg.format(fy="All", q=fiscal_quarter) + return files, backup_file_name, continue_msg + + def handle(self, *args, **options): + """Delete and reparse datafiles matching a query.""" + fiscal_year = options.get('fiscal_year', None) + fiscal_quarter = options.get('fiscal_quarter', None) + reparse_all = options.get('all', False) + selected_files = options.get('files', None) + selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None + new_indices = reparse_all is True + + # have to check if the selected_files is not None + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files + + if not args_passed: + logger.warning("No arguments supplied.") + self.print_help("manage.py", "clean_and_parse") + return + + # Set up the backup file name and continue message + files, backup_file_name, continue_msg = self.get_files_to_reparse( + fiscal_year, + fiscal_quarter, + selected_files, + reparse_all) + + # end of the if statement fmt_str = "be" if new_indices else "NOT be" continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str) diff --git a/tdrs-backend/tdpservice/templates/error_pages/500.html b/tdrs-backend/tdpservice/templates/error_pages/500.html new file mode 100644 index 0000000000..0060aa69fa --- /dev/null +++ b/tdrs-backend/tdpservice/templates/error_pages/500.html @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + Page not found - TANF Data Portal + + + + +
+ Skip to main content +
U.S. flag

An Official website of the United States government

+
+
+
+ +
+ +
+
+
+
+
+
+
+
+

Error

+

We’re sorry, there was an error in response.

+ +

{{ error }}.

+
+ + + Contact Us + +
+
+
+
+
+
+
+
+ +
+ + + \ No newline at end of file From 95b6ebdaa9738bdde2f3826d36f08e484ca535df Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 19 Sep 2024 09:03:58 -0400 Subject: [PATCH 28/36] linting and corrections --- .../tdpservice/data_files/admin/admin.py | 28 ++++---- .../static/admin/js/admin/mymodel.js | 72 ++++++------------- 2 files changed, 38 insertions(+), 62 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 5598c451b1..de9296452f 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -8,7 +8,6 @@ from django.conf import settings from django.utils.html import format_html from datetime import datetime, timedelta, timezone -from django.template.response import TemplateResponse DOMAIN = settings.FRONTEND_BASE_URL @@ -29,6 +28,8 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" class Media: + """Media class for DataFileAdmin.""" + js = ('admin/js/admin/mymodel.js',) actions = ['reparse_cmd'] @@ -40,20 +41,21 @@ def reparse_cmd(self, request, queryset): files = queryset.values_list("id", flat=True) file_ids = ",".join(map(str, files)) - number_of_files = queryset.count() - number_of_records = sum([df.summary.total_number_of_records_in_file for df in queryset]) - context = dict( - self.admin_site.each_context(request), - title="Are you sure?", - action="reparse_cmd", - queryset=queryset, - opts=self.model._meta, - msg=f"{number_of_files} datafiles, {number_of_records} records will be lost", - file_ids=file_ids, + from django.core.management import call_command + from django.utils.translation import ngettext + from django.contrib import messages + call_command("clean_and_reparse", f"-f {file_ids}") + self.message_user( + request, + ngettext( + "%d file successfully submitted for reparsing.", + "%d files successfully submitted for reparsing.", + files.count(), + ) + % files.count(), + messages.SUCCESS, ) - #return TemplateResponse(request, "admin/action_confirmation.html", context) - # TODO: add tests for this method def get_actions(self, request): """Return the actions.""" actions = super().get_actions(request) diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js index 1253151396..268358de09 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js @@ -1,19 +1,32 @@ $(window).on('load', function() { //your code here console.log('loaded'); - var S=document.querySelector('button[type=submit]'); - console.log(S); - // add the first listener + var S=document.querySelector('button[type=submit]'); // add the first listener var theForm = S.parentNode.parentNode; - console.log(theForm); + + for (var i = 0; i < theForm.childNodes.length; i++) { + console.log(theForm.childNodes[i].className) + if (theForm.childNodes[i].className == "actions") { + form_header = theForm.childNodes[i]; + break; + } + } + for (var i = 0; i < form_header.childNodes.length; i++) { + console.log(form_header.childNodes[i].className) + if (form_header.childNodes[i].className == "action-counter") { + number_of_files = form_header.childNodes[i]; + break; + } + } S.addEventListener('click', function(e) { e.preventDefault(); - console.log('submitting'); - disableFields(); // your own function - e.preventDefault(); - - alert("Ensure you input a value in both fields!"); - theForm.submit(); //this is working + disableFields(); + if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) { + console.log('submitting'); + theForm.submit(); + } else { + console.log('not submitting'); + }; }); @@ -23,42 +36,3 @@ $(window).on('load', function() { disableFields = function() { console.log('disabling fields'); } - - -reparseFiles = function() { - var xhr = new XMLHttpRequest(); - xhr.onreadystatechange = function () { - if (xhr.readyState === 4) { - alert(xhr.response); - } - } - xhr.open('POST', '/v1/data_files/run_action_reparse_cmd/', false); - xhr.setRequestHeader("X-CSRFToken", '{{ csrf_token }}'); - xhr.setRequestHeader("Content-Type", "application/json;charset=UTF-8"); - formData = new FormData(); - formData.append('file_ids', '{{ file_ids }}'); - data = {'file_ids': '{{ file_ids }}'}; - console.log(xhr) - xhr.send(JSON.stringify(data)); -} - -/* - -*/ From 1c19e87a5e279316289f346956faab16d403a7d9 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Thu, 19 Sep 2024 09:05:09 -0400 Subject: [PATCH 29/36] lint --- tdrs-backend/tdpservice/data_files/admin/admin.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index de9296452f..a74dfa717a 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -36,9 +36,6 @@ class Media: def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" - # TOTO: remove this if part. This is just for testing - # we can run the command directly here - files = queryset.values_list("id", flat=True) file_ids = ",".join(map(str, files)) from django.core.management import call_command From 98bd857db39bd0f43938423c3842eec9864362e5 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Fri, 20 Sep 2024 08:24:29 -0400 Subject: [PATCH 30/36] redirect to meta model --- tdrs-backend/tdpservice/data_files/admin/admin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index a74dfa717a..8214b13a71 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -8,6 +8,7 @@ from django.conf import settings from django.utils.html import format_html from datetime import datetime, timedelta, timezone +from django.shortcuts import redirect DOMAIN = settings.FRONTEND_BASE_URL @@ -52,6 +53,7 @@ def reparse_cmd(self, request, queryset): % files.count(), messages.SUCCESS, ) + return redirect("/admin/search_indexes/reparsemeta/") def get_actions(self, request): """Return the actions.""" From 3daebc532f5bdaf27ec9daf65b767af4555bed86 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 25 Sep 2024 14:31:47 -0400 Subject: [PATCH 31/36] 3106 replying comments --- .github/workflows/deploy-on-label.yml | 2 +- tdrs-backend/tdpservice/data_files/admin/admin.py | 2 +- .../js/admin/{mymodel.js => admin_datafile_model.js} | 11 +++-------- 3 files changed, 5 insertions(+), 10 deletions(-) rename tdrs-backend/tdpservice/data_files/static/admin/js/admin/{mymodel.js => admin_datafile_model.js} (80%) diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 65dc199bb4..8d4720f07c 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -71,7 +71,7 @@ jobs: - name: Circle CI Deployment Trigger id: curl-circle-ci - #if: steps.get-pr-state.outputs.STATE == 'success' + if: steps.get-pr-state.outputs.STATE == 'success' uses: promiseofcake/circleci-trigger-action@v1 with: user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }} diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 8214b13a71..52b0760c1d 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -31,7 +31,7 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): class Media: """Media class for DataFileAdmin.""" - js = ('admin/js/admin/mymodel.js',) + js = ('admin/js/admin/admin_datafile_model.js',) actions = ['reparse_cmd'] diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js similarity index 80% rename from tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js rename to tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js index 268358de09..24d9469745 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/mymodel.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js @@ -1,8 +1,8 @@ $(window).on('load', function() { //your code here console.log('loaded'); - var S=document.querySelector('button[type=submit]'); // add the first listener - var theForm = S.parentNode.parentNode; + var submitBtn=document.querySelector('button[type=submit]'); // add the first listener + var theForm = submitBtn.parentNode.parentNode; for (var i = 0; i < theForm.childNodes.length; i++) { console.log(theForm.childNodes[i].className) @@ -18,7 +18,7 @@ $(window).on('load', function() { break; } } - S.addEventListener('click', function(e) { + submitBtn.addEventListener('click', function(e) { e.preventDefault(); disableFields(); if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) { @@ -31,8 +31,3 @@ $(window).on('load', function() { }); - - -disableFields = function() { - console.log('disabling fields'); -} From f1c22f47996751b4d25f2546c3e3754f45a8e996 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 25 Sep 2024 15:06:26 -0400 Subject: [PATCH 32/36] move imports to the top --- tdrs-backend/tdpservice/data_files/admin/admin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 52b0760c1d..dd68fd3cc0 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -1,7 +1,6 @@ """Admin class for DataFile objects.""" from django.contrib import admin from tdpservice.core.utils import ReadOnlyAdminMixin -# from tdpservice.core.filters import custom_filter_title from tdpservice.data_files.models import DataFile, LegacyFileTransfer from tdpservice.parsers.models import DataFileSummary, ParserError from tdpservice.data_files.admin.filters import DataFileSummaryPrgTypeFilter, LatestReparseEvent, VersionFilter @@ -9,6 +8,9 @@ from django.utils.html import format_html from datetime import datetime, timedelta, timezone from django.shortcuts import redirect +from django.core.management import call_command +from django.utils.translation import ngettext +from django.contrib import messages DOMAIN = settings.FRONTEND_BASE_URL @@ -39,9 +41,6 @@ def reparse_cmd(self, request, queryset): """Reparse the selected data files.""" files = queryset.values_list("id", flat=True) file_ids = ",".join(map(str, files)) - from django.core.management import call_command - from django.utils.translation import ngettext - from django.contrib import messages call_command("clean_and_reparse", f"-f {file_ids}") self.message_user( request, From f3a39596f42ad7cf4c31eb6fcf517fc29b584245 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:12:52 -0400 Subject: [PATCH 33/36] Update admin.py: renamed reparse command --- tdrs-backend/tdpservice/data_files/admin/admin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index dd68fd3cc0..cf4c831f77 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -35,9 +35,9 @@ class Media: js = ('admin/js/admin/admin_datafile_model.js',) - actions = ['reparse_cmd'] + actions = ['reparse'] - def reparse_cmd(self, request, queryset): + def reparse(self, request, queryset): """Reparse the selected data files.""" files = queryset.values_list("id", flat=True) file_ids = ",".join(map(str, files)) @@ -58,8 +58,8 @@ def get_actions(self, request): """Return the actions.""" actions = super().get_actions(request) if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists(): - if "reparse_cmd" in actions: - del actions["reparse_cmd"] + if "reparse" in actions: + del actions["reparse"] return actions def status(self, obj): From d1fb4d66b206fedda64a00add16695558690abb5 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Mon, 30 Sep 2024 12:20:17 -0400 Subject: [PATCH 34/36] 3106 removed unused function --- .../data_files/static/admin/js/admin/admin_datafile_model.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js index 24d9469745..bcfd4e6a51 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js @@ -20,7 +20,6 @@ $(window).on('load', function() { } submitBtn.addEventListener('click', function(e) { e.preventDefault(); - disableFields(); if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) { console.log('submitting'); theForm.submit(); From 6e723e06530dfe3fcdd2244dc0dad40c287da292 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Mon, 30 Sep 2024 12:39:43 -0400 Subject: [PATCH 35/36] 3106 fixed reparsing command issues --- tdrs-backend/tdpservice/data_files/admin/admin.py | 3 +++ .../data_files/static/admin/js/admin/admin_datafile_model.js | 1 + 2 files changed, 4 insertions(+) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index cf4c831f77..995f3e8e9c 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -60,6 +60,9 @@ def get_actions(self, request): if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists(): if "reparse" in actions: del actions["reparse"] + else: + if "reparse" not in actions: + actions["reparse"] = (self.reparse, "reparse", "Reparse selected data files)") return actions def status(self, obj): diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js index bcfd4e6a51..709b52b097 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js @@ -2,6 +2,7 @@ $(window).on('load', function() { //your code here console.log('loaded'); var submitBtn=document.querySelector('button[type=submit]'); // add the first listener + console.log(submitBtn.parentNode); var theForm = submitBtn.parentNode.parentNode; for (var i = 0; i < theForm.childNodes.length; i++) { From a1c4631a4e6df39c308726acc6643c9cc76bd1d0 Mon Sep 17 00:00:00 2001 From: Mo Sohani Date: Wed, 2 Oct 2024 12:08:17 -0400 Subject: [PATCH 36/36] resolve merge conflict errors --- tdrs-backend/tdpservice/data_files/admin/admin.py | 3 +-- .../static/admin/js/admin/admin_datafile_model.js | 3 --- .../management/commands/clean_and_reparse.py | 8 ++++++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/admin/admin.py b/tdrs-backend/tdpservice/data_files/admin/admin.py index 995f3e8e9c..f17e0d6f11 100644 --- a/tdrs-backend/tdpservice/data_files/admin/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin/admin.py @@ -58,8 +58,7 @@ def get_actions(self, request): """Return the actions.""" actions = super().get_actions(request) if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists(): - if "reparse" in actions: - del actions["reparse"] + actions.pop("reparse", None) else: if "reparse" not in actions: actions["reparse"] = (self.reparse, "reparse", "Reparse selected data files)") diff --git a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js index 709b52b097..6c16846ce5 100644 --- a/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js +++ b/tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js @@ -2,18 +2,15 @@ $(window).on('load', function() { //your code here console.log('loaded'); var submitBtn=document.querySelector('button[type=submit]'); // add the first listener - console.log(submitBtn.parentNode); var theForm = submitBtn.parentNode.parentNode; for (var i = 0; i < theForm.childNodes.length; i++) { - console.log(theForm.childNodes[i].className) if (theForm.childNodes[i].className == "actions") { form_header = theForm.childNodes[i]; break; } } for (var i = 0; i < form_header.childNodes.length; i++) { - console.log(form_header.childNodes[i].className) if (form_header.childNodes[i].className == "action-counter") { number_of_files = form_header.childNodes[i]; break; diff --git a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py index 42988ea108..ab707335cf 100644 --- a/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py +++ b/tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py @@ -306,7 +306,10 @@ def handle(self, *args, **options): selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None new_indices = reparse_all is True - # have to check if the selected_files is not None + # Option that can only be specified by calling `handle` directly and passing it. + testing = options.get('testing', False) + ## + args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files if not args_passed: @@ -330,7 +333,8 @@ def handle(self, *args, **options): fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})" continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str) - self._handle_input(testing, continue_msg) + if not selected_files: + self._handle_input(testing, continue_msg) system_user, created = User.objects.get_or_create(username='system') if created: