Merge branch 'main' into rjm/2351-org-requests-page

.github/actions/django-security-check/README.md

@@ -38,7 +38,7 @@ jobs:
         id: check
         uses: victoriadrake/django-security-check@master
       - name: Upload output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: security-check-output
           path: output.txt

.github/workflows/security-check.yaml

@@ -44,7 +44,7 @@ jobs:
         id: check
         uses: ./.github/actions/django-security-check
       - name: Upload output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: security-check-output
           path: ./src/output.txt

docs/operations/data_migration.md

@@ -860,3 +860,55 @@ Example: `cf ssh getgov-za`
 
 ### Running locally
 ```docker-compose exec app ./manage.py populate_domain_request_dates```
+
+## Create federal portfolio
+This script takes the name of a `FederalAgency` (like 'AMTRAK') and does the following:
+1. Creates the portfolio record based off of data on the federal agency object itself.
+2. Creates suborganizations from existing DomainInformation records.
+3. Associates the SeniorOfficial record (if it exists).
+4. Adds this portfolio to DomainInformation / DomainRequests or both.
+
+Errors:
+1. ValueError: Federal agency not found in database.
+2. Logged Warning: No senior official found for portfolio
+3. Logged Error: No suborganizations found for portfolio.
+4. Logged Warning: No new suborganizations to add.
+5. Logged Warning: No valid DomainRequest records to update.
+6. Logged Warning: No valid DomainInformation records to update.
+
+### Running on sandboxes
+
+#### Step 1: Login to CloudFoundry
+```cf login -a api.fr.cloud.gov --sso```
+
+#### Step 2: SSH into your environment
+```cf ssh getgov-{space}```
+
+Example: `cf ssh getgov-za`
+
+#### Step 3: Create a shell instance
+```/tmp/lifecycle/shell```
+
+#### Step 4: Upload your csv to the desired sandbox
+[Follow these steps](#use-scp-to-transfer-data-to-sandboxes) to upload the federal_cio csv to a sandbox of your choice.
+
+#### Step 5: Running the script
+```./manage.py create_federal_portfolio "{federal_agency_name}" --both```
+
+Example (only requests): `./manage.py create_federal_portfolio "AMTRAK" --parse_requests`
+
+### Running locally
+
+#### Step 1: Running the script
+```docker-compose exec app ./manage.py create_federal_portfolio "{federal_agency_name}" --both```
+
+##### Parameters
+|   | Parameter                  | Description                                                                                |
+|:-:|:-------------------------- |:-------------------------------------------------------------------------------------------|
+| 1 | **federal_agency_name**    | Name of the FederalAgency record surrounded by quotes. For instance,"AMTRAK".              |
+| 2 | **both**                   | If True, runs parse_requests and parse_domains.                                            |
+| 3 | **parse_requests**         | If True, then the created portfolio is added to all related DomainRequests.                |
+| 4 | **parse_domains**          | If True, then the created portfolio is added to all related Domains.                       |
+
+Note: Regarding parameters #2-#3, you cannot use `--both` while using these. You must specify either `--parse_requests` or `--parse_domains` seperately. While all of these parameters are optional in that you do not need to specify all of them,
+you must specify at least one to run this script.

src/epplibwrapper/cert.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 
-from django.conf import settings
+from django.conf import settings  # type: ignore
 
 
 class Cert:
@@ -12,7 +12,7 @@ class Cert:
     variable but Python's ssl library requires a file.
     """
 
-    def __init__(self, data=settings.SECRET_REGISTRY_CERT) -> None:
+    def __init__(self, data=settings.SECRET_REGISTRY_CERT) -> None:  # type: ignore
         self.filename = self._write(data)
 
     def __del__(self):
@@ -31,4 +31,4 @@ class Key(Cert):
     """Location of private key as written to disk."""
 
     def __init__(self) -> None:
-        super().__init__(data=settings.SECRET_REGISTRY_KEY)
+        super().__init__(data=settings.SECRET_REGISTRY_KEY)  # type: ignore

src/registrar/admin.py

@@ -963,7 +963,9 @@ def change_view(self, request, object_id, form_url="", extra_context=None):
         domain_ids = user_domain_roles.values_list("domain_id", flat=True)
         domains = Domain.objects.filter(id__in=domain_ids).exclude(state=Domain.State.DELETED)
 
-        extra_context = {"domain_requests": domain_requests, "domains": domains}
+        portfolio_ids = obj.get_portfolios().values_list("portfolio", flat=True)
+        portfolios = models.Portfolio.objects.filter(id__in=portfolio_ids)
+        extra_context = {"domain_requests": domain_requests, "domains": domains, "portfolios": portfolios}
         return super().change_view(request, object_id, form_url, extra_context)
 
 

src/registrar/assets/js/get-gov-admin.js

@@ -748,7 +748,10 @@ function initializeWidgetOnList(list, parentId) {
 
             //------ Requested Domains
             const requestedDomainElement = document.getElementById('id_requested_domain');
-            const requestedDomain = requestedDomainElement.options[requestedDomainElement.selectedIndex].text;
+            // We have to account for different superuser and analyst markups
+            const requestedDomain = requestedDomainElement.options 
+                ? requestedDomainElement.options[requestedDomainElement.selectedIndex].text 
+                : requestedDomainElement.text;
 
             //------ Submitter
             // Function to extract text by ID and handle missing elements
@@ -762,7 +765,10 @@ function initializeWidgetOnList(list, parentId) {
             // Extract the submitter name, title, email, and phone number
             const submitterDiv = document.querySelector('.form-row.field-submitter');
             const submitterNameElement = document.getElementById('id_submitter');
-            const submitterName = submitterNameElement.options[submitterNameElement.selectedIndex].text;
+            // We have to account for different superuser and analyst markups
+            const submitterName = submitterNameElement 
+                ? submitterNameElement.options[submitterNameElement.selectedIndex].text 
+                : submitterDiv.querySelector('a').text;
             const submitterTitle = extractTextById('contact_info_title', submitterDiv);
             const submitterEmail = extractTextById('contact_info_email', submitterDiv);
             const submitterPhone = extractTextById('contact_info_phone', submitterDiv);

src/registrar/config/settings.py

@@ -23,6 +23,9 @@
 from pathlib import Path
 from typing import Final
 from botocore.config import Config
+import json
+import logging
+from django.utils.log import ServerFormatter
 
 # # #                          ###
 #      Setup code goes here      #
@@ -57,7 +60,7 @@
 env_debug = env.bool("DJANGO_DEBUG", default=False)
 env_is_production = env.bool("IS_PRODUCTION", default=False)
 env_log_level = env.str("DJANGO_LOG_LEVEL", "DEBUG")
-env_base_url = env.str("DJANGO_BASE_URL")
+env_base_url: str = env.str("DJANGO_BASE_URL")
 env_getgov_public_site_url = env.str("GETGOV_PUBLIC_SITE_URL", "")
 env_oidc_active_provider = env.str("OIDC_ACTIVE_PROVIDER", "identity sandbox")
 
@@ -192,7 +195,7 @@
     "registrar.registrar_middleware.CheckPortfolioMiddleware",
 ]
 
-# application object used by Django’s built-in servers (e.g. `runserver`)
+# application object used by Django's built-in servers (e.g. `runserver`)
 WSGI_APPLICATION = "registrar.config.wsgi.application"
 
 # endregion
@@ -415,7 +418,7 @@
 # and to interpret datetimes entered in forms
 TIME_ZONE = "UTC"
 
-# enable Django’s translation system
+# enable Django's translation system
 USE_I18N = True
 
 # enable localized formatting of numbers and dates
@@ -450,6 +453,40 @@
 #   logger.error("Can't do this important task. Something is very wrong.")
 #   logger.critical("Going to crash now.")
 
+
+class JsonFormatter(logging.Formatter):
+    """Formats logs into JSON for better parsing"""
+
+    def __init__(self):
+        super().__init__(datefmt="%d/%b/%Y %H:%M:%S")
+
+    def format(self, record):
+        log_record = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "name": record.name,
+            "lineno": record.lineno,
+            "message": record.getMessage(),
+        }
+        return json.dumps(log_record)
+
+
+class JsonServerFormatter(ServerFormatter):
+    """Formats server logs into JSON for better parsing"""
+
+    def format(self, record):
+        formatted_record = super().format(record)
+        log_entry = {"server_time": record.server_time, "level": record.levelname, "message": formatted_record}
+        return json.dumps(log_entry)
+
+
+# default to json formatted logs
+server_formatter, console_formatter = "json.server", "json"
+
+# don't use json format locally, it makes logs hard to read in console
+if "localhost" in env_base_url:
+    server_formatter, console_formatter = "django.server", "verbose"
+
 LOGGING = {
     "version": 1,
     # Don't import Django's existing loggers
@@ -469,19 +506,25 @@
             "format": "[{server_time}] {message}",
             "style": "{",
         },
+        "json.server": {
+            "()": JsonServerFormatter,
+        },
+        "json": {
+            "()": JsonFormatter,
+        },
     },
     # define where log messages will be sent;
     # each logger can have one or more handlers
     "handlers": {
         "console": {
             "level": env_log_level,
             "class": "logging.StreamHandler",
-            "formatter": "verbose",
+            "formatter": console_formatter,
         },
         "django.server": {
             "level": "INFO",
             "class": "logging.StreamHandler",
-            "formatter": "django.server",
+            "formatter": server_formatter,
         },
         # No file logger is configured,
         # because containerized apps

src/registrar/fixtures_users.py

@@ -56,6 +56,7 @@ class UserFixture:
             "username": "8f8e7293-17f7-4716-889b-1990241cbd39",
             "first_name": "Katherine",
             "last_name": "Osos",
+            "email": "[email protected]",
         },
         {
             "username": "70488e0a-e937-4894-a28c-16f5949effd4",
@@ -171,7 +172,7 @@ class UserFixture:
             "username": "91a9b97c-bd0a-458d-9823-babfde7ebf44",
             "first_name": "Katherine-Analyst",
             "last_name": "Osos-Analyst",
-            "email": "[email protected]",
+            "email": "kosos+1@truss.works",
         },
         {
             "username": "2cc0cde8-8313-4a50-99d8-5882e71443e8",