serverless.yml

##
## This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved. 
## Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: accounting@stevens.usc.edu
##
## The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
##

# todo: rename to mentorpal-upload-processor
service: mentorpal-upload-sm

# pin to only deploy with a specific Serverless version
frameworkVersion: '3'

plugins:
  - serverless-deployment-bucket
  - serverless-layers
  - serverless-domain-manager

custom:
  domain:
    dev: api.devmentorpal.org
    qa: api.qamentorpal.org
    prod: api.mentorpal.org
  customDomain:
    domainName: ${self:custom.domain.${opt:stage, 'dev'}}
    basePath: 'upload' # api root url becomes: api.mentorpal.org/upload
    # certificateName: '*.mentorpal.org' didnt work
    # certificateName: 'c6107db7-c2ef-4e85-a21f-bee7d1ac008a' didnt work either
    stage: ${self:provider.stage}
    endpointType: 'regional'
    apiType: rest
    createRoute53Record: true
    securityPolicy: tls_1_2
    # we might have multiple services, better to manually invoke `sls create_domain`
    autoDomain: false

  stages:
    dev:
      LOG_LEVEL: DEBUG
      IS_SENTRY_ENABLED: false
      # v2 is hardcoded, we could use fallback stages instead
      # hack: qa and prod are in different regions so deploy fails because
      # it cannot resolve other env variables from ssm.
      # That's why we provide an empty default value '', but this
      # in effect prevent sls from catching errors (missing variable in ssm)!
      S3_STATIC_ARN: ${ssm:/devmentorpal/s3_static_arn, ''}
      GRAPHQL_ENDPOINT: https://api.devmentorpal.org/graphql/graphql
      SECRET_HEADER_VALUE: ${ssm:/mentorpal/newdev/shared/secret_header_value, ''}
      SECRET_HEADER_NAME: ${ssm:/mentorpal/newdev/shared/secret_header_name, ''}
      JWT_SECRET: ${ssm:/mentorpal/newdev/shared/jwt_secret, ''}
      STATIC_URL_BASE: ${ssm:/mentorpal/newdev/shared/static_url_base, ''}
      ALERT_SNS_ARN: 'no alerts in dev'
      WEBACL_ARN: ${ssm:/devmentorpal/api_firewall_arn, ''}
    qa:
      S3_STATIC_ARN: ${ssm:/qamentorpal/s3_static_arn, ''}
      LOG_LEVEL: DEBUG
      IS_SENTRY_ENABLED: true
      GRAPHQL_ENDPOINT: https://api.qamentorpal.org/graphql/graphql
      JWT_SECRET: ${ssm:/mentorpal/v2/shared/jwt_secret, ''}
      SECRET_HEADER_VALUE: ${ssm:/mentorpal/v2/shared/secret_header_value, ''}
      SECRET_HEADER_NAME: ${ssm:/mentorpal/v2/shared/secret_header_name, ''}
      STATIC_URL_BASE: ${ssm:/mentorpal/v2/shared/static_url_base, ''}
      ALERT_SNS_ARN: ${ssm:/qamentorpal/shared/sns_alert_topic_arn, ''}
      WEBACL_ARN: ${ssm:/qamentorpal/api_firewall_arn, ''}
    prod:
      LOG_LEVEL: INFO
      IS_SENTRY_ENABLED: true
      GRAPHQL_ENDPOINT: https://api.mentorpal.org/graphql/graphql
      S3_STATIC_ARN: ${ssm:/mentorpal/s3_static_arn, ''}
      JWT_SECRET: ${ssm:/mentorpal/cf/shared/jwt_secret, ''}
      SECRET_HEADER_VALUE: ${ssm:/mentorpal/cf/shared/secret_header_value, ''}
      SECRET_HEADER_NAME: ${ssm:/mentorpal/cf/shared/secret_header_name, ''}
      ALERT_SNS_ARN: ${ssm:/mentorpal/shared/sns_alert_topic_arn, ''}
      STATIC_URL_BASE: ${ssm:/mentorpal/cf/shared/static_url_base, ''}
      WEBACL_ARN: ${ssm:/mentorpal/api_firewall_arn, ''}

  # serverless-layers requires a deployment bucket to be created before deploying this stack
  serverless-layers:
    - dependencies:
        layersDeploymentBucket: ${self:provider.deploymentBucket.name}
        dependenciesPath: ./requirements.txt
        compatibleRuntimes:
          - python3.9
        # applies to all lambdas

provider:
  name: aws
  region: ${opt:region, 'us-east-1'}
  stage: ${opt:stage, 'dev'} # stage is dev unless otherwise specified with --stage flag
  deploymentBucket:
    name: '${self:service}-sls-deploy-${self:provider.stage}'
    blockPublicAccess: true
    serverSideEncryption: AES256
    versioning: false
  stackTags:
    ENVIRONMENT: ${self:provider.stage}
    PROJECT: ${self:service}-${self:provider.stage}
    REPOSITORY: mentor-upload
  runtime: python3.9
  architecture: x86_64 # because of the static ffmpeg binaries and python dependencies
  endpointType: regional
  apiGateway:
    minimumCompressionSize: 1024
    # https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-payload-encodings.html
    binaryMediaTypes:
      - 'multipart/form-data'    
  tracing:
    lambda: true
    apiGateway: true
  logRetentionInDays: 30
  logs:
    restApi:
      # Enables HTTP access logs (default: true)
      accessLogging: true
      # # Log format to use for access logs
      # format: 'requestId: $context.requestId'
      # Enable execution logging (default: true)
      executionLogging: true
      level: INFO  # INFO or ERROR
      # Log full requests/responses for execution logging (default: true)
      fullExecutionData: false

  environment:
    STAGE: ${self:provider.stage}
    PYTHON_ENV: careerfair-${self:provider.stage}
    S3_STATIC_ARN: ${self:custom.stages.${self:provider.stage}.S3_STATIC_ARN}
    FFMPEG_EXECUTABLE: /opt/ffmpeg/ffmpeg
    FFPROBE_EXECUTABLE: /opt/ffmpeg/ffprobe
    GRAPHQL_ENDPOINT: ${self:custom.stages.${self:provider.stage}.GRAPHQL_ENDPOINT}
    IS_SENTRY_ENABLED: ${self:custom.stages.${self:provider.stage}.IS_SENTRY_ENABLED}
    STATIC_URL_BASE: ${self:custom.stages.${self:provider.stage}.STATIC_URL_BASE}
    SENTRY_DSN_MENTOR_UPLOAD: '${ssm:/mentorpal/upload/sentry_dsn}'
    TRANSCRIBE_INPUT_BUCKET: '${self:service}-transcribe-input-${self:provider.stage}'
    TRANSCRIBE_OUTPUT_BUCKET: '${self:service}-transcribe-output-${self:provider.stage}'
    SIGNED_UPLOAD_BUCKET: '${self:service}-signed-upload-${self:provider.stage}'
    SECRET_HEADER_NAME: ${self:custom.stages.${self:provider.stage}.SECRET_HEADER_NAME}
    SECRET_HEADER_VALUE: ${self:custom.stages.${self:provider.stage}.SECRET_HEADER_VALUE}
    # AWS_REGION is reserved
    REGION: ${self:provider.region}
  
  # iam permissions for all lambda functions
  iam:
    role:
      statements:
        - Effect: "Allow"
          Action:
            - "s3:PutObject"
            - "s3:GetObject"
          Resource:
            - '${self:custom.stages.${self:provider.stage}.S3_STATIC_ARN}/*'
            - 'arn:aws:s3:::${self:provider.environment.TRANSCRIBE_INPUT_BUCKET}/*'
            - 'arn:aws:s3:::${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}/*'
            - 'arn:aws:s3:::${self:provider.environment.SIGNED_UPLOAD_BUCKET}/*'
        - Effect: "Allow"
          Action:
            - "s3:ListBucket"
          Resource:
            - '${self:custom.stages.${self:provider.stage}.S3_STATIC_ARN}'
            - 'arn:aws:s3:::${self:provider.environment.TRANSCRIBE_INPUT_BUCKET}'
            - 'arn:aws:s3:::${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}'
            - 'arn:aws:s3:::${self:provider.environment.SIGNED_UPLOAD_BUCKET}'
        - Effect: Allow
          Action:
          - "transcribe:StartTranscriptionJob"
          Resource: "*"
        - Effect: "Allow"
          Action:
            - dynamodb:GetItem
            - dynamodb:PutItem
            - dynamodb:UpdateItem
          Resource:
            Fn::GetAtt: [JobsTable, Arn]
        - Effect: Allow
          Action:
            - states:StartExecution
            - states:SendTaskSuccess
            - states:SendTaskFailure
          Resource:
            - Fn::GetAtt: [AnswerUploadStepFunction, Arn]
package:
#  individually: false
 patterns:
    # exclude everything:
     - '!./**'
    # and then add back in only the files we need:
     - '*.py'
     - 'module/**'    

layers:
  # binaries are shared and this will make lambdas size smaller
  binaries:
    path: ./binaries
    package:
      patterns:
        - '!./**'
        # relative to layer path:
        # when attached, these are available under /opt
        - ./ffmpeg/ffmpeg
        - ./ffmpeg/ffprobe
        - './MediaInfo_DLL_21.09_Lambda/lib/**'
    name: binaries-layer-${self:provider.stage}
    description: Bundles ffmpeg and mediainfo binaries/libs
    compatibleArchitectures: # optional, a list of architectures this layer is compatible with
      - x86_64
    licenseInfo: GPLv3 # optional, a string specifying license information
    retain: false # If true, layer versions are not deleted as new ones are created

functions:
  authorizer_func:
    handler: authorizer.handler
    environment:
      JWT_SECRET: ${self:custom.stages.${self:provider.stage}.JWT_SECRET}

  http_thumbnail:
    handler: thumbnail.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    events:
      - http:
          path: /thumbnail
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_header:
    handler: header_upload.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    events:
      - http:
          path: /header
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_footer:
    handler: footer_upload.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    events:
      - http:
          path: /footer
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_vbg:
    handler: vbg_upload.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    events:
      - http:
          path: /vbg
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_vtt:
    handler: vtt_upload.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    events:
      - http:
          path: /vtt
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_transfer_start:
    handler: transfer-start.handler
    memorySize: 512
    timeout: 10
    environment:
      JOBS_TABLE_NAME: upload-jobs-${self:provider.stage}
    events:
      - http:
          path: /transfer/mentor
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token
  http_regen_vtt:
    handler: regen-vtt.handler
    memorySize: 512
    timeout: 10
    events:
      - http:
          path: /regen_vtt
          method: post
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token
    layers:
      # binaries gets named "Binaries"+"LambdaLayer":
      # see https://www.serverless.com/framework/docs/providers/aws/guide/layers#using-your-layers
      - { Ref: BinariesLambdaLayer }

  http_transfer_status:
    handler: transfer-status.handler
    memorySize: 256
    timeout: 10
    environment:
      JOBS_TABLE_NAME: upload-jobs-${self:provider.stage}
    events:
      - http:
          path: /transfer/status/{id}
          method: get
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token
          request:
            parameters:
              paths:
                id: true

  dbstream_transfer_process:
    description: Triggered by new records in dynamo, implements mentor transfer
    handler: transfer-process.handler
    memorySize: 2048
    timeout: 900 # max
    environment:
      JOBS_TABLE_NAME: upload-jobs-${self:provider.stage}
    events:
      - stream:
          type: dynamodb
          maximumRetryAttempts: 3
          arn:
            Fn::GetAtt: [JobsTable, StreamArn]
          # to avoid triggers on status update, make sure it's only when a new job is added:
          filterPatterns:
            - eventName: [INSERT]

  http_upload_url:
    handler: upload-url.handler
    memorySize: 256
    timeout: 10
    events:
      - http:
          path: /upload/url
          method: get
          cors: true
          authorizer:
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token

  http_answer_upload:
    handler: answer-upload.handler
    memorySize: 1024
    timeout: 30 # 30sec is max for http requests
    environment:
      ANSWER_UPLOAD_STEP_FUNCTION_ARN:
        Fn::GetAtt: [AnswerUploadStepFunction, Arn]
    events:
      - http:
          path: /upload/answer
          method: post
          cors: true
          authorizer: 
            name: authorizer_func
            resultTtlInSeconds: 300
            identitySource: method.request.header.Authorization
            type: token
    layers:
      # binaries gets named "Binaries"+"LambdaLayer":
      # see https://www.serverless.com/framework/docs/providers/aws/guide/layers#using-your-layers
      - { Ref: BinariesLambdaLayer }

  step_trim:
      handler: step-trim.handler
      memorySize: 4096 # needs cpu to run ffmpeg
      timeout: 600 # some videos take 5min or so
      layers:
        # binaries gets named "Binaries"+"LambdaLayer":
        # see https://www.serverless.com/framework/docs/providers/aws/guide/layers#using-your-layers
        - { Ref: BinariesLambdaLayer }

  step_transcode_web:
      handler: step-transcode-web.handler
      memorySize: 8192 # 5min video takes 100sec with 2GB, 50sec with 4GB, 30sec with 8GB
      timeout: 900 # give max time to make sure it gets transcoded
      layers:
        - { Ref: BinariesLambdaLayer }

  step_transcode_mobile:
      handler: step-transcode-mobile.handler
      memorySize: 8192 # 5min video takes 100sec with 2GB, 50sec with 4GB, 30sec with 8GB
      timeout: 900 # give max time to make sure it gets transcoded
      layers:
        - { Ref: BinariesLambdaLayer }

  step_transcribe_start:
      handler: step-transcribe-start.handler
      memorySize: 1024 # need to extract audio
      timeout: 30
      environment:
        TRANSCRIBE_INPUT_BUCKET: ${self:provider.environment.TRANSCRIBE_INPUT_BUCKET}
        TRANSCRIBE_OUTPUT_BUCKET: ${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}
      layers:
        # binaries gets named "Binaries"+"LambdaLayer":
        # see https://www.serverless.com/framework/docs/providers/aws/guide/layers#using-your-layers
        - { Ref: BinariesLambdaLayer }

  step_transcribe_collect:
    handler: step-transcribe-collect.handler
    memorySize: 512
    timeout: 30
    events:
      # only one suffix is allowed and we have .json and .vtt
      - s3:
          bucket: ${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}
          event: s3:ObjectCreated:*
          existing: true # otherwise sls will try to create it twice and fail
          rules:
            - suffix: '.json'
      - s3:
          bucket: ${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}
          event: s3:ObjectCreated:*
          existing: true # otherwise sls will try to create it twice and fail
          rules:
            - suffix: '.vtt'

  step_mark_failed:
      handler: step-mark-failed.handler
      timeout: 30

resources:
  Conditions:
    CreateCloudWatchAlarm:
      Fn::Or:
        - Fn::Equals: ['${self:provider.stage}', 'qa']
        - Fn::Equals: ['${self:provider.stage}', 'prod']
  Resources:
    AnswerUploadStepFunction: ${file(./resources/StepFunctions/AnswerUploadStepFunction.yml):StateMachine}
    AnswerUploadStepFunctionExecutionRole: ${file(./resources/StepFunctions/AnswerUploadStepFunctionExecutionRole.yml):Role}

    AnswerUploadDLQ:
      Type: AWS::SQS::Queue
      Properties:
        DelaySeconds: 30
        QueueName: answer-upload-dlq-${self:provider.stage}
        MessageRetentionPeriod: 1209600 # max 14 days, default is 4 days

    TranscribeInputBucket:
      Type: AWS::S3::Bucket
      Properties:
        BucketName: ${self:provider.environment.TRANSCRIBE_INPUT_BUCKET}
        LifecycleConfiguration:
          Rules:
          - Id: DeleteInputFileAfter7Days
            Status: Enabled
            ExpirationInDays: 7

    TranscribeOutputBucket:
      Type: AWS::S3::Bucket
      Properties:
        BucketName: ${self:provider.environment.TRANSCRIBE_OUTPUT_BUCKET}
        LifecycleConfiguration:
          Rules:
          - Id: DeleteResultAfter7Days
            Status: Enabled
            ExpirationInDays: 7

    AnswerUploadDLQAlarm:
      Type: AWS::CloudWatch::Alarm
      Condition: CreateCloudWatchAlarm
      Properties:
        ActionsEnabled: true
        AlarmName: 'Answer upload failed jobs ${self:resources.Resources.AnswerUploadDLQ.Properties.QueueName}'
        AlarmDescription: 'Number of failed answer upload jobs greater than zero'
        Namespace: 'AWS/SQS'
        Statistic: 'Sum'
        MetricName: 'ApproximateNumberOfMessagesVisible'
        ComparisonOperator: 'GreaterThanThreshold'
        EvaluationPeriods: 1
        Period: 300 # 5 minutes in seconds
        Threshold: 0
        TreatMissingData: 'notBreaching'
        Dimensions:
          - Name: QueueName
            Value: ${self:resources.Resources.AnswerUploadDLQ.Properties.QueueName}
        AlarmActions:
          - ${self:custom.stages.${self:provider.stage}.ALERT_SNS_ARN}
        OKActions:
          - ${self:custom.stages.${self:provider.stage}.ALERT_SNS_ARN}

    ApiErrors:
      Type: AWS::CloudWatch::Alarm
      Condition: CreateCloudWatchAlarm
      Properties:
        ActionsEnabled: true
        AlarmName: 'upload processor API Gateway 5xx errors ${self:provider.stage}'
        AlarmDescription: 'upload processor API Gateway 5xx errors ${self:provider.stage}'
        Namespace: AWS/ApiGateway
        MetricName: 5XXError
        Statistic: Sum
        Threshold: 0
        ComparisonOperator: 'GreaterThanThreshold'
        EvaluationPeriods: 1
        Period: 60
        TreatMissingData: 'notBreaching'
        Dimensions:
          - Name: ApiName
            Value: '${self:provider.stage}-${self:service}'
        AlarmActions:
          - ${self:custom.stages.${self:provider.stage}.ALERT_SNS_ARN}

    Response5xx:
      Type: 'AWS::ApiGateway::GatewayResponse'
      Properties:
        ResponseParameters:
          gatewayresponse.header.Access-Control-Allow-Origin: 'method.request.header.origin'
        ResponseType: DEFAULT_5XX
        RestApiId:
          Ref: 'ApiGatewayRestApi'
    Response4xx:
      Type: 'AWS::ApiGateway::GatewayResponse'
      Properties:
        ResponseParameters:
          gatewayresponse.header.Access-Control-Allow-Origin: 'method.request.header.origin'
        ResponseType: DEFAULT_4XX
        RestApiId:
          Ref: 'ApiGatewayRestApi'

    JobsTable:
      Type: AWS::DynamoDB::Table
      DeletionPolicy: Delete
      UpdateReplacePolicy: Delete
      Properties:
        TableName: upload-jobs-${self:provider.stage}
        BillingMode: PAY_PER_REQUEST
        StreamSpecification:
          StreamViewType: NEW_IMAGE
        AttributeDefinitions:
          - AttributeName: id
            AttributeType: S
        KeySchema:
          - AttributeName: id
            KeyType: HASH
        # to cleanup jobs after a while set this attribute as a number
        TimeToLiveSpecification: 
          AttributeName: ttl
          Enabled: true

    SignedUploadBucket:
      Type: AWS::S3::Bucket
      Properties:
        BucketName: ${self:provider.environment.SIGNED_UPLOAD_BUCKET}
        # videos are processed and uploaded to STATIC_S3 so clean them up:
        LifecycleConfiguration:
          Rules:
          - Id: DeleteResultAfter30Days
            Status: Enabled
            ExpirationInDays: 30
        CorsConfiguration:
          CorsRules:
            - AllowedMethods:
                - PUT
                - POST
              AllowedOrigins:
                - '*'
              AllowedHeaders:
                - '*'

    # this fails on first deploy because CloudFormation tries to create
    # association but the gateway does not yet exist
    FirewallAssociation:
      Type: AWS::WAFv2::WebACLAssociation
      DependsOn: "ApiGatewayRestApi"
      Properties:
        ResourceArn:
          Fn::Sub: 'arn:aws:apigateway:${AWS::Region}::/restapis/${ApiGatewayRestApi}/stages/${self:provider.stage}'
        WebACLArn: '${self:custom.stages.${self:provider.stage}.WEBACL_ARN}'