3106 reparse django action

tdrs-backend/tdpservice/data_files/admin/admin.py

@@ -1,13 +1,16 @@
 """Admin class for DataFile objects."""
 from django.contrib import admin
 from tdpservice.core.utils import ReadOnlyAdminMixin
-# from tdpservice.core.filters import custom_filter_title
 from tdpservice.data_files.models import DataFile, LegacyFileTransfer
 from tdpservice.parsers.models import DataFileSummary, ParserError
 from tdpservice.data_files.admin.filters import DataFileSummaryPrgTypeFilter, LatestReparseEvent, VersionFilter
 from django.conf import settings
 from django.utils.html import format_html
 from datetime import datetime, timedelta, timezone
+from django.shortcuts import redirect
+from django.core.management import call_command
+from django.utils.translation import ngettext
+from django.contrib import messages
 
 DOMAIN = settings.FRONTEND_BASE_URL
 
@@ -23,11 +26,42 @@ def has_change_permission(self, request, obj=None):
         """Read only permissions."""
         return False
 
-
 @admin.register(DataFile)
 class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin):
     """Admin class for DataFile models."""
 
+    class Media:
+        """Media class for DataFileAdmin."""
+
+        js = ('admin/js/admin/admin_datafile_model.js',)
+
+    actions = ['reparse']
+
+    def reparse(self, request, queryset):
+        """Reparse the selected data files."""
+        files = queryset.values_list("id", flat=True)
+        file_ids = ",".join(map(str, files))
+        call_command("clean_and_reparse", f"-f {file_ids}")
+        self.message_user(
+            request,
+            ngettext(
+                "%d file successfully submitted for reparsing.",
+                "%d files successfully submitted for reparsing.",
+                files.count(),
+            )
+            % files.count(),
+            messages.SUCCESS,
+        )
+        return redirect("/admin/search_indexes/reparsemeta/")
+
+    def get_actions(self, request):
+        """Return the actions."""
+        actions = super().get_actions(request)
+        if not request.user.groups.filter(name__in=["OFA System Admin", "OFA Admin"]).exists():
+            if "reparse" in actions:
+                del actions["reparse"]
+        return actions
+
     def status(self, obj):
         """Return the status of the data file summary."""
         return DataFileSummary.objects.get(datafile=obj).status

tdrs-backend/tdpservice/data_files/static/admin/js/admin/admin_datafile_model.js

@@ -0,0 +1,33 @@
+$(window).on('load', function() {
+    //your code here
+    console.log('loaded');
+    var submitBtn=document.querySelector('button[type=submit]');    // add the first listener
+    var theForm = submitBtn.parentNode.parentNode;
+
+    for (var i = 0; i < theForm.childNodes.length; i++) {
+        console.log(theForm.childNodes[i].className)
+        if (theForm.childNodes[i].className == "actions") {
+          form_header = theForm.childNodes[i];
+          break;
+        }        
+    }
+    for (var i = 0; i < form_header.childNodes.length; i++) {
+        console.log(form_header.childNodes[i].className)
+        if (form_header.childNodes[i].className == "action-counter") {
+          number_of_files = form_header.childNodes[i];
+          break;
+        }        
+    }
+    submitBtn.addEventListener('click', function(e) {
+        e.preventDefault();
+        disableFields();
+        if (confirm("You are about to re-parse " + number_of_files.innerHTML.split(/(\s+)/)[0] + " files. Are you sure you want to continue?")) {
+            console.log('submitting');
+            theForm.submit();
+        } else {
+            console.log('not submitting');
+        };
+    });
+
+
+});

tdrs-backend/tdpservice/data_files/views.py

@@ -22,6 +22,8 @@
 from tdpservice.data_files.s3_client import S3Client
 from tdpservice.parsers.models import ParserError
 from tdpservice.parsers.serializers import ParsingErrorSerializer
+from rest_framework.parsers import JSONParser, FormParser
+from django.core.management import call_command
 
 logger = logging.getLogger(__name__)
 
@@ -151,6 +153,13 @@ def download_error_report(self, request, pk=None):
         serializer = ParsingErrorSerializer(parser_errors, many=True, context=self.get_serializer_context())
         return Response(get_xls_serialized_file(serializer.data))
 
+    @action(methods=["post"], detail=False, parser_classes=[JSONParser, MultiPartParser, FormParser])
+    def run_action_reparse_cmd(self, request, pk=None):
+        """Run the reparse command."""
+        # Need to add the reparsing file ids to the request object
+        call_command("clean_and_reparse", f"-f {request.data['file_ids']}")
+        return Response({'status': 'success'})
+
 
 class GetYearList(APIView):
     """Get list of years for which there are data_files."""

tdrs-backend/tdpservice/search_indexes/management/commands/clean_and_reparse.py

@@ -32,6 +32,7 @@ def add_arguments(self, parser):
         parser.add_argument("-y", "--fiscal_year", type=int, help="Reparse all files in the fiscal year, e.g. 2021.")
         parser.add_argument("-a", "--all", action='store_true', help="Clean and reparse all datafiles. If selected, "
                             "fiscal_year/quarter aren't necessary.")
+        parser.add_argument("-f", "--files", nargs='+', type=str, help="Re-parse specific datafiles by datafile id")
 
     def __get_log_context(self, system_user):
         """Return logger context."""
@@ -223,28 +224,20 @@ def __calculate_timeout(self, num_files, num_records):
         logger.info(f"Setting timeout for the reparse event to be {delta} seconds from meta model creation date.")
         return delta
 
-    def handle(self, *args, **options):
-        """Delete and reparse datafiles matching a query."""
-        fiscal_year = options.get('fiscal_year', None)
-        fiscal_quarter = options.get('fiscal_quarter', None)
-        reparse_all = options.get('all', False)
-        new_indices = reparse_all is True
-
-        args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all
-
-        if not args_passed:
-            logger.warn("No arguments supplied.")
-            self.print_help("manage.py", "clean_and_parse")
-            return
-
+    def get_files_to_reparse(case, fiscal_year, fiscal_quarter, selected_files, reparse_all):
+        """Get the files to reparse."""
         backup_file_name = "/tmp/reparsing_backup"
         files = DataFile.objects.all()
         continue_msg = "You have selected to reparse datafiles for FY {fy} and {q}. The reparsed files "
+        if selected_files:
+            files = files.filter(id__in=selected_files)
+            backup_file_name += "_selected_files"
+            continue_msg = continue_msg.format(fy="All", q="Q1-4")
         if reparse_all:
             backup_file_name += "_FY_All_Q1-4"
             continue_msg = continue_msg.format(fy="All", q="Q1-4")
         else:
-            if not fiscal_year and not fiscal_quarter:
+            if not fiscal_year and not fiscal_quarter and not selected_files:
                 print(
                     'Options --fiscal_year and --fiscal_quarter not set. '
                     'Provide either option to continue, or --all to wipe all submissions.'
@@ -262,6 +255,33 @@ def handle(self, *args, **options):
                 files = files.filter(quarter=fiscal_quarter)
                 backup_file_name += f"_FY_All_{fiscal_quarter}"
                 continue_msg = continue_msg.format(fy="All", q=fiscal_quarter)
+        return files, backup_file_name, continue_msg
+
+    def handle(self, *args, **options):
+        """Delete and reparse datafiles matching a query."""
+        fiscal_year = options.get('fiscal_year', None)
+        fiscal_quarter = options.get('fiscal_quarter', None)
+        reparse_all = options.get('all', False)
+        selected_files = options.get('files', None)
+        selected_files = [int(file) for file in selected_files[0].split(',')] if selected_files else None
+        new_indices = reparse_all is True
+
+        # have to check if the selected_files is not None
+        args_passed = fiscal_year is not None or fiscal_quarter is not None or reparse_all or selected_files
+
+        if not args_passed:
+            logger.warning("No arguments supplied.")
+            self.print_help("manage.py", "clean_and_parse")
+            return
+
+        # Set up the backup file name and continue message
+        files, backup_file_name, continue_msg = self.get_files_to_reparse(
+            fiscal_year,
+            fiscal_quarter,
+            selected_files,
+            reparse_all)
+
+        # end of the if statement
 
         fmt_str = "be" if new_indices else "NOT be"
         continue_msg += "will {new_index} stored in new indices and the old indices ".format(new_index=fmt_str)
@@ -270,10 +290,11 @@ def handle(self, *args, **options):
         fmt_str = f"ALL ({num_files})" if reparse_all else f"({num_files})"
         continue_msg += "\nThese options will delete and reparse {0} datafiles.".format(fmt_str)
 
-        c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower()
-        if c not in ['y', 'yes']:
-            print('Cancelled.')
-            return
+        if not selected_files:
+            c = str(input(f'\n{continue_msg}\nContinue [y/n]? ')).lower()
+            if c not in ['y', 'yes']:
+                print('Cancelled.')
+                return
 
         system_user, created = User.objects.get_or_create(username='system')
         if created: