diff --git a/.github/actions/setup-dependencies-macos/action.yml b/.github/actions/setup-dependencies-macos/action.yml index 06eedebe..8562f49d 100644 --- a/.github/actions/setup-dependencies-macos/action.yml +++ b/.github/actions/setup-dependencies-macos/action.yml @@ -26,11 +26,9 @@ runs: pip install -r requirements.txt pip install pytest pip install selenium - pip uninstall -y numpy - pip install numpy==1.26.4 - + - name: Download OPA executable shell: bash run: | python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }} - chmod +x opa_darwin_amd64 \ No newline at end of file + chmod +x opa_darwin_amd64 diff --git a/.github/actions/setup-dependencies-windows/action.yml b/.github/actions/setup-dependencies-windows/action.yml index 771ee2a8..5025e459 100644 --- a/.github/actions/setup-dependencies-windows/action.yml +++ b/.github/actions/setup-dependencies-windows/action.yml @@ -18,7 +18,7 @@ runs: pip install virtualenv python -m venv .venv .venv\Scripts\activate - + - name: Install dependencies shell: powershell run: | @@ -26,16 +26,7 @@ runs: pip install -r requirements.txt pip install pytest pip install selenium - pip uninstall -y numpy - pip install numpy==1.26.4 - - # Below python v3.9, a lower numpy v1.24.4 is used - #$pythonVersion = [version]${{ inputs.python-version }} - #if ($pythonVersion -ge [version]"3.8.18") { - # pip uninstall -y numpy - # pip install numpy==1.26.4 - #} - + - name: Download OPA executable shell: powershell - run: python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }} \ No newline at end of file + run: python download_opa.py -v ${{ inputs.opa-version }} -os ${{ inputs.operating-system }} diff --git a/.github/workflows/run_smoke_test.yml b/.github/workflows/run_smoke_test.yml index c913a5d7..eee99664 100644 --- a/.github/workflows/run_smoke_test.yml +++ b/.github/workflows/run_smoke_test.yml @@ -1,31 +1,76 @@ name: Run Smoke Test on: - workflow_call: - workflow_dispatch: pull_request: types: [opened, reopened] branches: - "main" + pull_request_review: + types: [submitted] push: - # Uncomment when testing locally - #paths: - # - ".github/workflows/run_smoke_test.yml" - # - ".github/actions/setup-dependencies-windows/action.yml" - # - ".github/actions/setup-dependencies-macos/action.yml" + # These paths are primarily for workflow testing purposes + paths: + - ".github/workflows/run_smoke_test.yml" + - ".github/actions/setup-dependencies-windows/action.yml" + - ".github/actions/setup-dependencies-macos/action.yml" branches: - "main" + - "*smoketest*" + workflow_call: + workflow_dispatch: + inputs: + operating-system: + description: "Choose operating system(s), format must be an array of strings:" + required: true + type: string + default: "['windows-latest', 'macos-latest']" + python-version: + description: "Choose python version(s), format must be an array of strings:" + required: true + type: string + default: "['3.10']" + opa-version: + description: "Choose OPA version" + required: true + type: string + default: "0.60.0" jobs: + configuration: + runs-on: ubuntu-latest + outputs: + operating-system: ${{ steps.variables.outputs.operating-system }} + python-version: ${{ steps.variables.outputs.python-version }} + opa-version: ${{ steps.variables.outputs.opa-version }} + steps: + - name: Configure variable outputs + id: variables + run: | + # For manual runs + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + operatingsystem_val="${{ inputs.operating-system }}" + pythonversion_val="${{ inputs.python-version }}" + opaversion_val="${{ inputs.opa-version }}" + + # Default values for other events + else + operatingsystem_val="['windows-latest', 'macos-latest']" + pythonversion_val="['3.10']" + opaversion_val="0.60.0" + fi + echo "operating-system=$operatingsystem_val" >> "$GITHUB_OUTPUT" + echo "python-version=$pythonversion_val" >> "$GITHUB_OUTPUT" + echo "opa-version=$opaversion_val" >> "$GITHUB_OUTPUT" + smoke-test: + needs: configuration strategy: fail-fast: false matrix: - os: [windows-latest, macos-latest] + operating-system: ${{ fromJSON(needs.configuration.outputs.operating-system) }} # See https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json, # ctrl + f and search "python-3..-" for supported versions - python-version: ["3.9", "3.12"] # "3.8 fails with numpy uninstall" - runs-on: ${{ matrix.os }} - environment: Development + python-version: ${{ fromJSON(needs.configuration.outputs.python-version) }} + runs-on: ${{ matrix.operating-system }} steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -38,19 +83,19 @@ jobs: cache-dependency-path: "requirements.txt" - name: Setup Dependencies (Windows) - if: ${{ matrix.os == 'windows-latest' }} + if: ${{ matrix.operating-system == 'windows-latest' }} uses: ./.github/actions/setup-dependencies-windows with: operating-system: "windows" - opa-version: "0.60.0" + opa-version: ${{ needs.configuration.outputs.opa-version }} python-version: ${{ matrix.python-version }} - name: Setup Dependencies (macOS) - if: ${{ matrix.os == 'macos-latest' }} + if: ${{ matrix.operating-system == 'macos-latest' }} uses: ./.github/actions/setup-dependencies-macos with: operating-system: "macos" - opa-version: "0.60.0" + opa-version: ${{ needs.configuration.outputs.opa-version }} python-version: ${{ matrix.python-version }} - name: Setup credentials for service account @@ -61,4 +106,4 @@ jobs: json: ${{ secrets.GWS_GITHUB_AUTOMATION_CREDS }} - name: Run ScubaGoggles and check for correct output - run: pytest -s -vvv ./Testing/Functional/SmokeTests/ --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --domain="${{ secrets.GWS_DOMAIN }}" + run: pytest ./Testing/Functional/SmokeTests/ -vvv --subjectemail="${{ secrets.GWS_SUBJECT_EMAIL }}" --customerdomain="${{ secrets.GWS_DOMAIN }}" diff --git a/Testing/Functional/SmokeTests/README.md b/Testing/Functional/SmokeTests/README.md new file mode 100644 index 00000000..268b6179 --- /dev/null +++ b/Testing/Functional/SmokeTests/README.md @@ -0,0 +1,142 @@ +# ScubaGoggles Functional Smoke Testing Automation +The ScubaGoggles repository consists of an automation suite to help test the functionality of the ScubaGoggles tool itself. The test automation is geared towards contributors who want to execute the functional smoke testing orchestrator as part of their development/testing activity. + +This README outlines the ScubaGoggles software test automation structure and its usage. The document also contains instructions for adding new tests to the existing automation suite if necessary. + +## Table of Contents +- [Smoke Testing Prerequisites](#smoke-testing-prerequisites) + - [Pytest and Selenium](#pytest-and-selenium) + - [Google Service Account](#google-service-account) +- [Functional Smoke Testing Structure](#functional-smoke-testing-structure) + - [Smoke Testing Classes and Methods](#smoke-testing-classes-and-methods) + - [Automated workflow via GitHub Actions](#automated-workflow-via-github-actions) +- [Functional Smoke Testing Usage](#functional-smoke-testing-usage) + - [Running in a Local Development Environment](#running-in-a-local-development-environment) + - [Running Remotely via GitHub Actions](#running-remotely-via-github-actions) +- [Adding New Tests](#adding-new-tests) + +## Smoke Testing Prerequisites ## +Running the ScubaGoggles functional smoke tests requires a Windows, MacOS, or Linux computer or VM. The development environment should have Python v3.10.x installed at a minimum ([refer to our installing Python dependencies documentation if its not already installed](https://github.com/cisagov/ScubaGoggles/blob/main/docs/installation/DownloadAndInstall.md#installing-python-dependencies)), Pytest, and Selenium installed locally. + +### Pytest and Selenium ### +Pytest is a Python testing framework which is commonly used for unit, integration, and functional testing. ([Pytest Get Started](https://docs.pytest.org/en/stable/getting-started.html)) + +Selenium supports automation of all the major browsers in the market through the use of WebDriver. ([Selenium Get Started](https://www.selenium.dev/documentation/webdriver/getting_started/)) + +To install Pytest and Selenium on your development environment, open a new terminal session and run the following command: + +``` +pip install pytest selenium +``` + +> [!NOTE] +> The functional smoke tests use Chrome as its WebDriver when running Selenium tests. [Setup ChromeDriver](https://developer.chrome.com/docs/chromedriver/get-started) if you don't already have the Google Chrome web browser installed. + +### Google Service Account ### +The ScubaGoggles functional smoke tests must be executed with a service account. [Refer to our service account documentation on how to get setup.](https://github.com/cisagov/ScubaGoggles/blob/main/docs/authentication/ServiceAccount.md#using-a-service-account) + +A `credentials.json` file is required at the root directory of the ScubaGoggles project if running the functional smoke tests in a local development environment. + +Take note of the `subjectemail`, the email used to authenticate with GWS that has necessary administrator permissions, and the GWS `customerdomain` that ScubaGoggles is run against. Both credentials are required in a later step. + +## Functional Smoke Testing Structure ## +ScubaGoggles functional smoke testing has two main components: the smoke testing orchestrator and the automated workflow run via GitHub Actions. + +### Smoke Testing Classes and Methods ### +The smoke testing orchestrator ([/Testing/Functional/SmokeTests/smoke_test.py](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/SmokeTests/smoke_test.py)) executes each test declared inside the `SmokeTest` class. The tests currently cover: +- if the `scubagoggles gws` command generates valid output for all baselines +- if ScubaResults.json contains API errors or exceptions +- if the generated baseline reports, i.e. BaselineReports.html, CalendarReport.html, ChatReport.html, etc., contain valid content and all links redirect accordingly + +The smoke testing utils ([/Testing/Functional/SmokeTests/smoke_test_utils.py](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/SmokeTests/smoke_test_utils.py)) stores helper methods which perform various operations. + +The Selenium Browser class ([/Testing/Functional/SmokeTests/selenium_browser.py](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/SmokeTests/selenium_browser.py)) encapsulates the setup, usage, and teardown of Selenium WebDriver instances. + +The Pytest configuration methods ([/Testing/Functional/SmokeTests/conftest.py](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/conftest.py)) declare various Pytest fixtures, allowing for the use of CLI arguments when invoking the Pytest command. + +### Automated Workflow via GitHub Actions ### +The automated workflow for running the functional smoke tests ([/.github/workflows/run_smoke_test.yml](https://github.com/cisagov/ScubaGoggles/blob/main/.github/workflows/run_smoke_test.yml)) is triggered on `push` events to the main branch, `pull_request` events when a pull request is opened/reopened/reviewed, and manually with custom user input via workflow_dispatch. + +## Functional Smoke Testing Usage ## +After completing all of the prerequisite steps, the functional smoke tests can be run in a local development environment or remotely via GitHub Actions. + +### Running in a Local Development Environment ### +> [!IMPORTANT] +> Ensure that you have correctly setup a Google service account and that the `credentials.json` stored at the root directory of the ScubaGoggles project is up to date. If you haven't already, please refer back to the [prerequisite step on Google Service Accounts](#google-service-account) for how to setup before proceeding. + +The following arguments are required when running the functional smoke tests: +- `--subjectemail="user@domain.com"` (the email used to authenticate with GWS, must have necessary administrator permissions) +- `--customerdomain="domain.com"` (the domain that ScubaGoggles is run against) + +Replace `user@domain.com` with your email and `domain.com` with your domain, then run the following command to execute the functional smoke tests: +``` +pytest ./Testing/Functional/SmokeTests/ -vvv --subjectemail="user@domain.com" --customerdomain="domain.com" +``` + +Common Pytest parameters and their use cases: +- `-v` or `--verbose` (shows individual test names and results) +- `-vv` (increases verbosity further, shows detailed output about each test) +- `-vvv` (shows even more detailed output and debug-level information) +- `-s` (disables output capturing allowing print() statements and logs to be shown in the console) +- `-k` (run tests that match a keyword) + + Example (only runs test_scubagoggles_output, deselects the rest): + ``` + pytest ./Testing/Functional/SmokeTests/ -vvv -k test_scubagoggles_output --subjectemail="user@domain.com" --customerdomain="domain.com" + ``` + +- `--tb=short`, `tb=long`, or `tb=no` (provide either brief, full, or suppress the traceback output for failed tests) +- `-q` (reduces output to show only minimal information) + +Run `pytest -h` for a full list of CLI options or [learn more about Pytest usage here.](https://docs.pytest.org/en/7.1.x/how-to/usage.html) + +### Running Remotely via GitHub Actions ### +Go to the [run_smoke_test.yml workflow](https://github.com/cisagov/ScubaGoggles/actions/workflows/run_smoke_test.yml) in the GitHub Actions tab, then click the "Run workflow" dropdown button. + +The default values are the following: +- ref branch: `main` but can be set to any branch +- operating system: `['windows-latest', 'macos-latest']` ([list of supported GitHub-hosted runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories)) +- python version: `['3.10']` +- opa version: "0.60.0" + +![Screenshot (226)](https://github.com/user-attachments/assets/6f25b7a9-3981-4866-a413-93df4bae1130) + +Feel free to play around with the inputs then click the "Run workflow" button when ready. The workflow will create a matrix strategy for each combination. For example, passing `['windows-latest', 'macos-latest']`, `['3.10', '3.11', 3.12']`, and OPA version `0.60.0` will create the following: + +![Screenshot (218)](https://github.com/user-attachments/assets/212b4e4b-d552-4dc9-a3f6-7f0e29accc4b) + +Some factors to consider: +- Each input is required so an empty string will fail validation. `[]`, `['']`, `['', ]` may also cause the workflow to error out, although this is expected behavior. +- `ubuntu-latest` has not been tested as a value for operating system. Support can be added for this, although its dependent on if this is something we want to test for ScubaGoggles as a whole. +- Python versions <3.10.x are not supported and will cause the smoke test workflow to fail. +- [Due to the lack of an array input type from GitHub](https://github.com/orgs/community/discussions/11692), the required format is an array of strings for the operating system and python version inputs. This is something to capture as a future todo once arrays are available. + +## Adding New Tests ## +A new smoke test should be added as a method in the [SmokeTest class](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/SmokeTests/smoke_test.py). Helper methods should be added in [smoke_test_utils.py](https://github.com/cisagov/ScubaGoggles/blob/main/Testing/Functional/SmokeTests/smoke_test_utils.py). + +Below is an example that tests the `scubagoggles gws` command: + +``` +class SmokeTest: + ... + + def test_scubagoggles_execution(self, subjectemail): + """ + Test if the `scubagoggles gws` command succeeds or fails. + + Args: + subjectemail: The email address of an admin user who created the service account + """ + try: + command: str = f"scubagoggles gws --subjectemail {subjectemail} --quiet" + result = subprocess.run(command, shell=True, check=True, capture_output=True) + + if result.returncode != 0: + print(f"Scubagoggles execution failed with error:\n{result.stderr}") + assert False + else: + print("Scubagoggles execution succeeded") + print(f"Output:\n{result.stdout}") + except Exception as e: + pytest.fail(f"An error occurred, {e}") +``` diff --git a/Testing/Functional/SmokeTests/selenium_browser.py b/Testing/Functional/SmokeTests/selenium_browser.py index 69bf9d9d..d3ffbc74 100644 --- a/Testing/Functional/SmokeTests/selenium_browser.py +++ b/Testing/Functional/SmokeTests/selenium_browser.py @@ -13,6 +13,7 @@ class Browser: def __init__(self): chrome_options = Options() chrome_options.add_argument("--headless") + chrome_options.add_argument("--window-size=1200,800") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument("--disable-dev-shm-usage") diff --git a/Testing/Functional/SmokeTests/smoke_test.py b/Testing/Functional/SmokeTests/smoke_test.py index 8c9c6839..3774e20c 100644 --- a/Testing/Functional/SmokeTests/smoke_test.py +++ b/Testing/Functional/SmokeTests/smoke_test.py @@ -57,7 +57,7 @@ def test_scubaresults(self): except (ValueError, Exception) as e: pytest.fail(f"An error occurred, {e}") - def test_scubagoggles_report(self, browser, domain): + def test_scubagoggles_report(self, browser, customerdomain): """ Test if the generated baseline reports are correct, i.e. BaselineReports.html, CalendarReport.html, ChatReport.html @@ -66,7 +66,7 @@ def test_scubagoggles_report(self, browser, domain): output_path: str = get_output_path() report_path: str = prepend_file_protocol(os.path.join(output_path, BASELINE_REPORTS)) browser.get(report_path) - run_selenium(browser, domain) + run_selenium(browser, customerdomain) except (ValueError, AssertionError, Exception) as e: browser.quit() pytest.fail(f"An error occurred, {e}") diff --git a/Testing/Functional/SmokeTests/smoke_test_utils.py b/Testing/Functional/SmokeTests/smoke_test_utils.py index 7073750c..f90ed528 100644 --- a/Testing/Functional/SmokeTests/smoke_test_utils.py +++ b/Testing/Functional/SmokeTests/smoke_test_utils.py @@ -11,7 +11,7 @@ from scubagoggles.utils import get_package_version OUTPUT_DIRECTORY = "GWSBaselineConformance" -BASELINE_REPORT_H1 = "SCuBA GWS Security Baseline Conformance Reports" +BASELINE_REPORT_H1 = "SCuBA GWS Secure Configuration Baseline Reports" CISA_GOV_URL = "https://www.cisa.gov/scuba" SCUBAGOGGLES_BASELINES_URL = "https://github.com/cisagov/ScubaGoggles/tree/main/baselines" @@ -125,13 +125,13 @@ def verify_scubaresults(jsonfile): if summary["Errors"] != 0: raise ValueError(f"{product} contains errors in the report") -def run_selenium(browser, domain): +def run_selenium(browser, customerdomain): """ Run Selenium tests against the generated reports. Args: browser: A Selenium WebDriver instance - domain: The user's domain + customerdomain: The customer domain """ verify_navigation_links(browser) h1 = browser.find_element(By.TAG_NAME, "h1").text @@ -149,9 +149,9 @@ def run_selenium(browser, domain): if len(reports_table) == 10: for i in range(len(reports_table)): - # Check if domain is present in agency table + # Check if customerdomain is present in agency table # Skip tool version if assessing the parent report - verify_tenant_table(browser, domain, True) + verify_tenant_table(browser, customerdomain, True) reports_table = get_reports_table(browser)[i] baseline_report = reports_table.find_elements(By.TAG_NAME, "td")[0] @@ -169,8 +169,8 @@ def run_selenium(browser, domain): h1 = browser.find_element(By.TAG_NAME, "h1").text assert h1 == products[product]["title"] - # Check if domain and tool version are present in individual report - verify_tenant_table(browser, domain, False) + # Check if customerdomain and tool version are present in individual report + verify_tenant_table(browser, customerdomain, False) policy_tables = browser.find_elements(By.TAG_NAME, "table") for table in policy_tables[1:]: @@ -239,14 +239,14 @@ def get_reports_table(browser): .find_elements(By.TAG_NAME, "tr") ) -def verify_tenant_table(browser, domain, parent): +def verify_tenant_table(browser, customerdomain, parent): """ Get the tenant table rows elements from the DOM. - (Table at the top of each report with user domain, baseline/tool version) + (Table at the top of each report with customer domain, baseline/tool version) Args: browser: A Selenium WebDriver instance - domain: The user's domain + customerdomain: The customer domain parent: boolean to determine parent/individual reports """ tenant_table_rows = ( @@ -255,8 +255,8 @@ def verify_tenant_table(browser, domain, parent): .find_elements(By.TAG_NAME, "tr") ) assert len(tenant_table_rows) == 2 - customer_domain = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[0].text - assert customer_domain == domain + domain = tenant_table_rows[1].find_elements(By.TAG_NAME, "td")[0].text + assert domain == customerdomain if not parent: # Check for correct tool version, e.g. 0.2.0 diff --git a/Testing/Functional/conftest.py b/Testing/Functional/conftest.py index 456f9be3..f84c2a62 100644 --- a/Testing/Functional/conftest.py +++ b/Testing/Functional/conftest.py @@ -13,7 +13,7 @@ def pytest_addoption(parser): parser: An instance of "argparse.ArgumentParser" """ parser.addoption("--subjectemail", action="store") - parser.addoption("--domain", action="store") + parser.addoption("--customerdomain", action="store") @pytest.fixture def subjectemail(pytestconfig): @@ -26,14 +26,14 @@ def subjectemail(pytestconfig): return pytestconfig.getoption("subjectemail") @pytest.fixture -def domain(pytestconfig): +def customerdomain(pytestconfig): """ - Setup code that shares the "domain" parameter across tests. + Setup code that shares the "customerdomain" parameter across tests. Args: pytestconfig: Provides access to the "Config" object for a current test session """ - return pytestconfig.getoption("domain") + return pytestconfig.getoption("customerdomain") @pytest.fixture def browser():