Action inputs to dispatch n-runs of a single functional test

.github/workflows/run-tests.yml

@@ -13,6 +13,45 @@ on:
       commit:
         description: "Commit SHA"
         required: true
+      run_single_functional_test:
+        description: "Whether to run a single test. If so, the rest of the input fields are required."
+        type: boolean
+        default: false
+      cassandra:
+        description: "cassandra"
+        type: boolean
+        default: false
+      sqlite:
+        description: "sqlite"
+        type: boolean
+        default: false
+      mysql8:
+        description: "mysql8"
+        type: boolean
+        default: false
+      postgres12:
+        description: "postgres12"
+        type: boolean
+        default: false
+      n_runs:
+        description: "[Single Test Only] Number of times to repeat the single test per database type"
+        type: number
+        default: 1
+      test_name:
+        description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
+        type: string
+      timeout_minutes:
+        description: "[Single Test Only] Test timeout in minutes"
+        type: number
+        default: 120
+      test_runner:
+        description: "[Single Test Only] Which runner to use. Choose higher RAM if your n_runs is high."
+        type: choice
+        default: "16GB RAM (ubuntu-20.04)"
+        options:
+          - "16GB RAM (ubuntu-20.04)"
+          - "64GB RAM (ubuntu-22.04)"
+
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -60,13 +99,16 @@ jobs:
       - run: make build-tests
 
   unit-test:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
     needs: misc-checks
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -87,13 +129,16 @@ jobs:
         run: make upload-test-results
 
   integration-test:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
     needs: misc-checks
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -123,21 +168,75 @@ jobs:
         if: ${{ !cancelled() }}
         run: make upload-test-results
 
+      - name: Tear down docker compose
+        if: ${{ always() }}
+        run: |
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
+
+
+  set-up-functional-test:
+    name: Set up functional test
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      runs_on: ${{ steps.generate_output.outputs.runs_on }}
+      dbs: ${{ steps.generate_output.outputs.dbs }}
+    steps:
+      - id: generate_output
+        run: |
+          shards=3
+          timeout=30
+          runs_on='["ubuntu-20.04"]'
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
+            shards=1
+            timeout=${{ inputs.timeout_minutes }}
+            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
+              runs_on='["ubuntu-latest-16-cores"]'
+            fi 
+            dbs_arr=()
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
+            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
+            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
+            if [[ ${#dbs_arr[@]} == 0 ]]; then
+              dbs='["sqlite"]'
+            else
+              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
+            fi
+          fi
+          {
+            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
+            echo "shards=$shards"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
+            echo "single_test_args=$single_test_args"
+            echo "runs_on=$runs_on"
+            echo "dbs=$dbs" 
+          } >> "$GITHUB_OUTPUT"
+      - id: cat_output
+        run: |
+          cat "$GITHUB_OUTPUT"
+
   functional-test:
     name: Functional test
-    needs: misc-checks
+    needs: [misc-checks, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ubuntu-20.04]
+        runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }}
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         name:
           - cass_es
           - cass_es8
           - sqlite
           - mysql8
           - postgres12
           - postgres12_pgx
-        shard_index: [0, 1, 2]
         include:
           - name: cass_es
             persistence_type: nosql
@@ -167,38 +266,52 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: 3
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
+      - name: Print functional test
+        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
+      - uses: ScribeMD/[email protected]
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
       - uses: actions/checkout@v4
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
 
       - uses: actions/setup-go@v5
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' }}
+        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         uses: hoverkraft-tech/[email protected]
         with:
           compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
           services: "${{ join(matrix.containers, '\n') }}"
           down-flags: -v
 
       - name: Run functional test
-        timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         run: make upload-test-results
 
   functional-test-xdc:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test xdc
     needs: misc-checks
     strategy:
@@ -232,6 +345,7 @@ jobs:
     env:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -260,6 +374,7 @@ jobs:
         run: make upload-test-results
 
   functional-test-ndc:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test ndc
     needs: misc-checks
     strategy:
@@ -324,6 +439,7 @@ jobs:
         run: make functional-test-ndc-coverage
 
   test-status:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Test Status
     needs:
       - unit-test
@@ -332,7 +448,6 @@ jobs:
       - functional-test-xdc
       - functional-test-ndc
     runs-on: ubuntu-20.04
-    if: always()
     env:
       RESULTS: ${{ toJSON(needs.*.result) }}
     steps:

Makefile

@@ -64,7 +64,7 @@ define NEWLINE
 
 endef
 
-TEST_TIMEOUT := 25m
+TEST_TIMEOUT ?= 25m
 
 PROTO_ROOT := proto
 PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")
@@ -391,8 +391,10 @@ pre-build-functional-test-coverage: prepare-coverage-test
 
 functional-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
+	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) \
+		 -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) \
+		 $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
 
 functional-test-xdc-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..."

develop/upload-test-results.sh

@@ -21,5 +21,6 @@ for file in *.junit.xml; do
     -F "run_env[branch]=${GITHUB_REF}" \
     -F "run_env[commit_sha]=${GITHUB_SHA}" \
     -F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+    -F "run_env[message]=${BUILDKITE_MESSAGE}" \
     https://analytics-api.buildkite.com/v1/uploads
-done
+done