Merge pull request #1577 from aiven/feature/use-aws

[Schedule merge 19.12] Feature/use aws
aiven · Dec 19, 2022 · 2828531 · 2828531
2 parents 7c28cd0 + c332d98
commit 2828531
Show file tree

Hide file tree

Showing 22 changed files with 14,149 additions and 142 deletions.
diff --git a/.env-local b/.env-local
@@ -0,0 +1,2 @@
+API_URL_SEARCH="<API URL for search from development AWS account>"
+API_URL_CREATE_FEEDBACK="<API URL for creating feedback from development AWS account>"
diff --git a/.github/workflows/sam-pipeline.yml b/.github/workflows/sam-pipeline.yml
@@ -0,0 +1,68 @@
+name: SAM Validate, Build, Test, Deploy
+on:
+  push:
+    branches:
+      - main
+      - feature/use-aws
+    paths:
+      - aws/**
+      - .github/workflows/sam-pipeline.yml
+jobs:
+  build-deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - uses: aws-actions/setup-sam@v2
+      - uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+            role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+            aws-region: ${{ secrets.AWS_REGION }}  
+      - name: SAM Validate
+        run: |
+          sam validate -t aws/template.yaml
+      - name: Configure variables
+        shell: bash
+        id: vars
+        env:
+          REPO: ${{ github.repository }}
+          HASH: ${{ github.sha }}
+          REF: ${{ github.ref }}
+        run: |
+          # Set variables
+          REPOSITORY=`echo $REPO | tr "/" "-"`
+          VERSIONKEY=${{ secrets.AWS_VERSION_KEY }}
+          ENVIRONMENT=$REPOSITORY-$VERSIONKEY-${{ secrets.AWS_REGION }}
+          # In this step we are setting variables and persistenting them
+          # into the environment so that they can be utilized in other steps
+          echo "repository=$REPOSITORY" >> $GITHUB_OUTPUT
+          echo "versionkey=$VERSION_KEY" >> $GITHUB_OUTPUT
+          echo "environment=$ENVIRONMENT" >> $GITHUB_OUTPUT
+          # Output variables to ensure their values are set correctly when ran
+          echo "The region is ${{ secrets.AWS_REGION }}"
+          echo "The repository is $REPOSITORY"
+          echo "The environment is $ENVIRONMENT"
+          echo "The version key is $VERSIONKEY"
+      # sam build
+      - name: SAM Build
+        run: sam build -t aws/template.yaml
+
+# Run Unit tests- Specify unit tests here 
+
+# sam deploy
+      - name: SAM Deploy
+        run: |
+          # Run SAM Deploy
+          sam deploy --debug --template-file .aws-sam/build/template.yaml \
+            --stack-name ${{ steps.vars.outputs.environment }} \
+            --s3-bucket ${{ steps.vars.outputs.environment }} \
+            --parameter-overrides \
+              'ParameterKey=OpenSearchURL,ParameterValue=${{ secrets.ES_URL }} \
+              ParameterKey=PostgresURL,ParameterValue=${{ secrets.PG_URL }} \
+              ParameterKey=PostgresCert,ParameterValue=${{ secrets.CA_CERT }} \
+              ParameterKey=Version,ParameterValue=${{ steps.vars.outputs.version }}' \
+            --no-fail-on-empty-changeset \
+            --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ node_modules
 env/
 venv/
 .venv/
+.env
diff --git a/_templates/feedback-form.html b/_templates/feedback-form.html
@@ -56,20 +56,20 @@
     justify-content: center;
   }
 
-  .updown-container #thumbsup:checked ~ label.up {
+  .updown-container #thumbsup:checked~label.up {
     background-color: #9de7ae;
   }
 
-  .updown-container #thumbsup:checked ~ label.down {
+  .updown-container #thumbsup:checked~label.down {
     background-color: #ddd;
     fill: #bbb;
   }
 
-  .updown-container #thumbsdown:checked ~ label.down {
+  .updown-container #thumbsdown:checked~label.down {
     background-color: #f0aeb4;
   }
 
-  .updown-container #thumbsdown:checked ~ label.up {
+  .updown-container #thumbsdown:checked~label.up {
     background-color: #ddd;
     fill: #bbb;
   }
@@ -84,62 +84,35 @@
 </style>
 
 <div class="feedback">
-  <form
-    id="feedback-form"
-    name="feedback"
-    method="POST"
-    data-netlify="true"
-    netlify-honeypot="bot-field"
-  >
+  <form id="feedback-form" name="feedback" method="POST">
     <div class="title-vote-container">
       <div>
         <span class="feedback-form-title">Did you find this useful?</span>
       </div>
       <div class="updown-container">
         <input id="thumbsup" type="radio" name="vote" value="up" />
         <input id="thumbsdown" type="radio" name="vote" value="down" />
-        <label for="thumbsup" aria-label="Thumbs up" class="up"
-          ><svg
-            width="32px"
-            height="32px"
-            viewBox="0 0 32 32"
-            xmlns="http://www.w3.org/2000/svg"
-          >
+        <label for="thumbsup" aria-label="Thumbs up" class="up"><svg width="32px" height="32px" viewBox="0 0 32 32"
+            xmlns="http://www.w3.org/2000/svg">
             <path
-              d="M 16.6875 3 L 16.375 3.28125 L 9.59375 10 L 5 10 L 5 26 L 21.84375 26 C 23.253906 26 24.484375 25.003906 24.78125 23.625 L 26.9375 13.625 C 27.332031 11.777344 25.886719 10 24 10 L 18.25 10 L 18.4375 9.25 C 18.640625 9.09375 18.769531 9.027344 19.0625 8.625 C 19.53125 7.984375 20 6.992188 20 5.65625 C 20 4.230469 18.710938 3 17.09375 3 Z M 17.40625 5.09375 C 17.828125 5.175781 18 5.347656 18 5.65625 C 18 6.558594 17.726563 7.117188 17.46875 7.46875 C 17.210938 7.820313 17.03125 7.90625 17.03125 7.90625 L 16.6875 8.09375 L 16.5625 8.5 L 15.96875 10.75 L 15.65625 12 L 24 12 C 24.660156 12 25.105469 12.574219 24.96875 13.21875 L 22.84375 23.21875 C 22.742188 23.6875 22.320313 24 21.84375 24 L 11 24 L 11 11.40625 Z M 7 12 L 9 12 L 9 24 L 7 24 Z"
-            /></svg
-        ></label>
-        <label for="thumbsdown" aria-label="Thumbs down" class="down"
-          ><svg
-            width="32px"
-            height="32px"
-            viewBox="0 0 32 32"
-            xmlns="http://www.w3.org/2000/svg"
-          >
+              d="M 16.6875 3 L 16.375 3.28125 L 9.59375 10 L 5 10 L 5 26 L 21.84375 26 C 23.253906 26 24.484375 25.003906 24.78125 23.625 L 26.9375 13.625 C 27.332031 11.777344 25.886719 10 24 10 L 18.25 10 L 18.4375 9.25 C 18.640625 9.09375 18.769531 9.027344 19.0625 8.625 C 19.53125 7.984375 20 6.992188 20 5.65625 C 20 4.230469 18.710938 3 17.09375 3 Z M 17.40625 5.09375 C 17.828125 5.175781 18 5.347656 18 5.65625 C 18 6.558594 17.726563 7.117188 17.46875 7.46875 C 17.210938 7.820313 17.03125 7.90625 17.03125 7.90625 L 16.6875 8.09375 L 16.5625 8.5 L 15.96875 10.75 L 15.65625 12 L 24 12 C 24.660156 12 25.105469 12.574219 24.96875 13.21875 L 22.84375 23.21875 C 22.742188 23.6875 22.320313 24 21.84375 24 L 11 24 L 11 11.40625 Z M 7 12 L 9 12 L 9 24 L 7 24 Z" />
+          </svg></label>
+        <label for="thumbsdown" aria-label="Thumbs down" class="down"><svg width="32px" height="32px"
+            viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg">
             <path
-              d="M 10.15625 6 C 8.746094 6 7.515625 6.996094 7.21875 8.375 L 5.0625 18.375 C 4.667969 20.222656 6.113281 22 8 22 L 13.75 22 L 13.5625 22.75 C 13.359375 22.90625 13.230469 22.972656 12.9375 23.375 C 12.46875 24.015625 12 25.007813 12 26.34375 C 12 27.769531 13.289063 29 14.90625 29 L 15.3125 29 L 15.625 28.71875 L 22.40625 22 L 27 22 L 27 6 Z M 10.15625 8 L 21 8 L 21 20.59375 L 14.59375 26.90625 C 14.171875 26.824219 14 26.652344 14 26.34375 C 14 25.441406 14.273438 24.882813 14.53125 24.53125 C 14.789063 24.179688 14.96875 24.09375 14.96875 24.09375 L 15.3125 23.90625 L 15.4375 23.5 L 16.03125 21.25 L 16.34375 20 L 8 20 C 7.339844 20 6.894531 19.425781 7.03125 18.78125 L 9.15625 8.78125 C 9.257813 8.3125 9.679688 8 10.15625 8 Z M 23 8 L 25 8 L 25 20 L 23 20 Z"
-            />
+              d="M 10.15625 6 C 8.746094 6 7.515625 6.996094 7.21875 8.375 L 5.0625 18.375 C 4.667969 20.222656 6.113281 22 8 22 L 13.75 22 L 13.5625 22.75 C 13.359375 22.90625 13.230469 22.972656 12.9375 23.375 C 12.46875 24.015625 12 25.007813 12 26.34375 C 12 27.769531 13.289063 29 14.90625 29 L 15.3125 29 L 15.625 28.71875 L 22.40625 22 L 27 22 L 27 6 Z M 10.15625 8 L 21 8 L 21 20.59375 L 14.59375 26.90625 C 14.171875 26.824219 14 26.652344 14 26.34375 C 14 25.441406 14.273438 24.882813 14.53125 24.53125 C 14.789063 24.179688 14.96875 24.09375 14.96875 24.09375 L 15.3125 23.90625 L 15.4375 23.5 L 16.03125 21.25 L 16.34375 20 L 8 20 C 7.339844 20 6.894531 19.425781 7.03125 18.78125 L 9.15625 8.78125 C 9.257813 8.3125 9.679688 8 10.15625 8 Z M 23 8 L 25 8 L 25 20 L 23 20 Z" />
           </svg>
         </label>
       </div>
     </div>
-    <p class="hidden">
-      <label
-        >Don't fill this out if you're human: <input name="bot-field"
-      /></label>
-    </p>
     <div class="message-submit-container hidden">
       <p>
-        <label
-          >Anything specific you want to share? (Please don't share any personal
+        <label>Anything specific you want to share? (Please don't share any personal
           details here. If you need a response,
           <a href="mailto:[email protected]">[email protected]</a> is the best way
-          to reach us.)<br /><textarea
-            class="feedback-message"
-            name="message"
-            rows="5"
-          ></textarea>
+          to reach us.)<br /><textarea class="feedback-message" name="message" rows="5"></textarea>
         </label>
+        <input type="text" id="honeypot_password" name="honeypot_password" style="display:none !important" tabindex="-1" autocomplete="off">
       </p>
       <p>
         <button type="submit">Send</button>
@@ -171,19 +144,22 @@
       event.preventDefault();
 
       var formData = new FormData(form.get()[0]);
+      formData.append("url", window.location.href);
 
-      fetch("/", {
-        method: "POST",
-        headers: { "Content-Type": "application/x-www-form-urlencoded" },
-        body: new URLSearchParams(formData).toString(),
-      })
+      fetch("{{ api_url_create_feedback }}", {
+          method: "POST",
+          body: JSON.stringify(Object.fromEntries(formData))
+        })
         .then(function (response) {
           if (response.status >= 400) {
             throw response;
           }
           $(".feedback").html($("#thankyou").html());
         })
         .catch(function (error) {
+          console.log({
+            error
+          });
           alert("Could not submit feedback, please try again later.");
         });
     }

diff --git a/_templates/search.html b/_templates/search.html
@@ -92,15 +92,18 @@ <h2>Search Results</h2>
     var searchResults = $("#search-results .search");
     var pagination = $("#search-results .pagination");
 
-    $.get("/.netlify/functions/search?" + $.param({ query: query, page: page }))
+    $.get("{{ api_url_search }}?" + $.param({
+        query: query,
+        page: page
+      }))
       .done(function (data) {
         if (data.resultCount === 0) {
           searchSummary.html("Your search did not match any documents.");
         } else {
           searchSummary.html(
             "Search finished, found " +
-              data.resultCount +
-              " page(s) matching the search query."
+            data.resultCount +
+            " page(s) matching the search query."
           );
         }
 

diff --git a/aws/.gitignore b/aws/.gitignore
@@ -0,0 +1,7 @@
+node_modules
+*.local.json
+packaged.yaml
+.aws-sam/
+samconfig.toml
+events/*.json
+*.pem
diff --git a/aws/CreateFeedback.md b/aws/CreateFeedback.md
@@ -0,0 +1,57 @@
+# Aiven Docs submitting feedback
+
+Feedback are submitted to a PostgreSQL database maintained by Aiven. Production URI can be found in 1Password under vault Aiven DevPortal.
+
+## Data
+
+Currently the feedback table has 4 columns:
+
+- **referrer** (string): Page URL where the feedback is submitted. Automatically filled
+- **vote** (string): "up" / "down". Either a upvote or downvote. Filled by user when clicking corresponding buttons.
+- **message** (string): Feedback field. Filled by user.
+- **created_at** (string): Timestamp of the submitted feedback. Automatically filled.
+
+The JS script in `feedback-form.html` would pick up `referrer`, `vote` and `message`by itself while `created_at`is assigned by the lambda upon creating the feedback in the database.
+
+## Testing locally
+
+### One time testing
+
+Following the instructions [here](https://github.com/aiven/devportal/tree/feature/use-aws/aws#development) one should have a `event.CreateFeedbackFunction.local.json` after running `npm run generate-event POST CreateFeedbackFunction`. It looks like this
+
+```
+{
+	"body":  "eyJ0ZXN0IjoiYm9keSJ9",
+	"resource":  "/{proxy+}",
+	"path":  "/path/to/resource",
+	"httpMethod":  "POST",
+}
+```
+
+To invoke this API with valid parameters, one should edit the `body` to be a stringified object containing necessary properties mentioned in the section above, e.g. `"body": "{\"url\":\"http://localhost:7777/\",\"vote\":\"down\",\"message\":\"test comment from sam local invoke\"}",`
+
+### Continuous testing
+
+Another way for continuous testing is to have a `env.local.json` with `PG_URL` and `CA_CERT` properly filled. Then one can run `npm run watch` and `npm run start` (in different terminals) and test it with Postman or other API testing tools.
+
+Be aware that when running SAM locally, it could mess with Sphynx autobuild `make livehtml`, causing it to reload repeatedly. Recommendation for now is to use API testing tools when developing the lambda, and only test with the local documentation site when needed.
+
+**Notes**
+
+- The `PG_URL` copied from the Aiven service should have the `sslmode=true` part removed e.g. `postgres://avnadmin:[email protected]:12691/defaultdb`.
+
+## Gotchas
+
+- When I test with the local doc site, submitting a feedback gives error about CORS?
+  - It's likely the URL of the local doc site isn't included in the allowed list of the lambda. To fix this, open the `create-feedback.js`and update the `allowedOrigins` to include your site then deploy it.
+
+```
+const  allowedOrigins  =  [
+	"https://devportal.pages.dev",
+	"http://localhost:[0-9]*",
+	"<Your local doc site URL>"
+];
+```
+
+- When I deploy the first time, I got some error related to samconfig syntax?
+  - There could be many causes, but one notable is wrong `CA_CERT` property format. It should look like `PostgresCert=\"-----BEGIN CERTIFICATE-----\n......\n-----END CERTIFICATE-----\n\""`
diff --git a/aws/README.md b/aws/README.md
@@ -0,0 +1,72 @@
+# Overview
+
+## Directory structure
+
+This project contains source code and supporting files for a serverless application that you can deploy with the AWS Serverless Application Model (AWS SAM) command line interface (CLI). It includes the following files and folders:
+
+- `src` - Code for the application's Lambda function.
+- `events` - Invocation events that you can use to invoke the function.
+- `__tests__` - Unit tests for the application code.
+- `template.yaml` - A template that defines the application's AWS resources e.g. functions
+
+# Preparation
+
+- Install [Docker](https://www.docker.com/) and [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html). We need Docker running in the background for AWS SAM to work. We don't need to interact directly with it.
+  - **Notes**: At the moment it seems some new versions of Docker may not work well with AWS SAM CLI, the latest safe version was [**4.7.0**](https://docs.docker.com/desktop/release-notes/#docker-desktop-470). It's possible newer versions may not have this issue. Recommended to install the latest first, then if AWS SAM complains about no Docker found, revert to the older version.
+- Run `npm i` inside this directory
+- Create a `env.local.json` file based on the existing `env.template.json`. This file helps run the API server with appropriate parameters during development. Production env are available in 1Password under vault Aiven DevPortal.
+  - For example, `ES_URL` and `PG_URL` can be obtained directly from the corresponding development Aiven services (OpenSearch & PostgreSQL)
+  - **Notes**: The `CA_CERT` for development can be obtained by going to the Aiven PostgreSQL service, download the CA cert into `./aws` directory
+  - ![Screenshot 2022-11-09 at 15 19 30](https://user-images.githubusercontent.com/110401626/200845923-0023847b-5f0d-45ef-ba19-d91975faeb3c.png)
+  - then run `npm run get-cert <cert file name e.g. ca.pem>`, and paste to that field.
+- Create a `.env` file in the root directory storing values for `API_URL_SEARCH` and `API_URL_CREATE_FEEDBACK`. These values will be used later for testing with local documentation site (can be left empty for now or filled with production value).
+
+# Development & Production
+
+Functions are to be modified inside `src/handlers` directory similar to a Node.js project. For "secrets" like DB connection, it uses environment variables. They are defined in `template.yaml` under `Globals > Environment > Variables` then later used in the function (e.g. `process.env.PG_URL`). This way we can separate different environments, keep the "secrets" secured and don't require additional services from AWS like SSM.
+
+## Development
+
+### One time testing
+
+Commands to use:
+
+- `npm run generate-event <API METHOD> <Function name>`: This generates an event for a function we declared in `template.yaml` e.g. `SearchFunction`, then we can modify the event with different parameters for one-time testing.
+- `npm run invoke <Function name>`: To be used with the `npm run generate-event` above. This will make a call to the API with parameters defined in the corresponding `event.[].local.json`.
+
+Please check out more details on doing one time testing for the existing functions here:
+
+- [Search function](https://github.com/aiven/devportal/blob/feature/use-aws/aws/SEARCH.md)
+- [Creating feedback function](https://github.com/aiven/devportal/blob/feature/use-aws/aws/CreateFeedback.md)
+
+### Continuous testing
+
+Commands to use:
+
+- `npm run watch`: Start a nodemon instance to watch for changes in `js,json,yaml` and run `sam build` when needed.
+- `npm run start`: Run this in another terminal window. This creates a server with configurations declared in `env.local.json` above, so we can test the API with Postman / browser / etc.
+
+During development, we may want to supply our own services e.g. OpenSearch / PostgreSQL / etc. through `env.local.json` so it doesn't conflict with production instances.
+
+### Testing with local documentation site
+
+With the server running, one can also test the API inside the local documentation site. This is done by changing the API URL inside their corresponding `.env` files to match the local API URL.
+
+From there, one can run `make livehtml` in the root folder to serve the doc site, and test the functions directly there.
+
+## Production
+
+When merging with master/main branch, GitHub Actions will take care of the deployment to AWS "aivenmkt" account (428637134338) with [OpenID Connect (OIDC)](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers). Note that this workflow only triggers when there's change in `aws` folder. Secret key are added in GitHub and available in 1Password under vault Aiven DevPortal.
+
+### Access to AWS "aivenmkt" account (428637134338)
+
+Request via [JIRA Access Management](https://aiven.atlassian.net/jira/software/c/projects/AM/boards/26). 
+You will receive a Slack message from Hallmonitor with your credentials. Once you have created a password and configured MFA, you will be able to login to the AWS Console and [assume a role in the aivenmkt account](https://signin.aws.amazon.com/switchrole?roleName=AivenMktDocsManagementUserRole&account=428637134338).
+
+## Workflow
+
+A simple workflow for improving the existing SearchFunction would be:
+
+- Run `npm run generate-event GET SearchFunction` to simulate event for the function (and change it if needed)
+- Run `npm run invoke SearchFunction` for one-time testings
+- Run `npm run start` and `npm run watch` to test / develop