AWS Amplify won't deploy because of CustomAuthTriggerResource error in CloudFormation

[fikisipi]

How did you install the Amplify CLI?

npm

If applicable, what version of Node.js are you using?

v18.15.0

Amplify CLI Version

12.7.0

What operating system are you using?

MacOS Sonoma M1

Did you make any manual changes to the cloud resources managed by Amplify? Please describe the changes made.

Deleted authTriggerFn7FCFA449

Describe the bug

When I created a pre-signup hook for Amplify, the following CloudFormation template was generated and it worked:

{
  "Description": "Custom Resource stack for Auth Trigger created using Amplify CLI",
  "AWSTemplateFormatVersion": "2010-09-09",
  "Parameters": {
    "env": {
      "Type": "String"
    },
    "userpoolId": {
      "Type": "String"
    },
    "userpoolArn": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupName": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupArn": {
      "Type": "String"
    },
    "functionmyappwebappPreSignupLambdaExecutionRole": {
      "Type": "String"
    }
  },
  "Conditions": {
    "ShouldNotCreateEnvResources": {
      "Fn::Equals": [
        {
          "Ref": "env"
        },
        "NONE"
      ]
    }
  },
  "Resources": {
    "UserPoolPreSignUpLambdaInvokePermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "functionmyappwebappPreSignupName"
        },
        "Principal": "cognito-idp.amazonaws.com",
        "SourceArn": {
          "Ref": "userpoolArn"
        }
      }
    },
    "authTriggerFnServiceRole08093B67": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "ManagedPolicyArns": [
          {
            "Fn::Join": [
              "",
              [
                "arn:",
                {
                  "Ref": "AWS::Partition"
                },
                ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
              ]
            ]
          }
        ]
      }
    },
    "authTriggerFnServiceRoleDefaultPolicyEC9285A8": {
      "Type": "AWS::IAM::Policy",
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "cognito-idp:DescribeUserPool",
                "cognito-idp:UpdateUserPool"
              ],
              "Effect": "Allow",
              "Resource": {
                "Ref": "userpoolArn"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "Roles": [
          {
            "Ref": "authTriggerFnServiceRole08093B67"
          }
        ]
      }
    },
    "authTriggerFn7FCFA449": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "ZipFile": "const response = require('cfn-response');\nconst aws = require('aws-sdk');\n\nexports.handler = async function (event, context) {\n  const physicalResourceId =\n    event.RequestType === 'Update' ? event.PhysicalResourceId : `${event.LogicalResourceId}-${event.ResourceProperties.userpoolId}`;\n\n  try {\n    const userPoolId = event.ResourceProperties.userpoolId;\n    const { lambdaConfig } = event.ResourceProperties;\n    const config = {};\n    const cognitoClient = new aws.CognitoIdentityServiceProvider();\n    const userPoolConfig = await cognitoClient.describeUserPool({ UserPoolId: userPoolId }).promise();\n    const userPoolParams = userPoolConfig.UserPool;\n    // update userPool params\n\n    const updateUserPoolConfig = {\n      UserPoolId: userPoolParams.Id,\n      Policies: userPoolParams.Policies,\n      SmsVerificationMessage: userPoolParams.SmsVerificationMessage,\n      AccountRecoverySetting: userPoolParams.AccountRecoverySetting,\n      AdminCreateUserConfig: userPoolParams.AdminCreateUserConfig,\n      AutoVerifiedAttributes: userPoolParams.AutoVerifiedAttributes,\n      EmailConfiguration: userPoolParams.EmailConfiguration,\n      EmailVerificationMessage: userPoolParams.EmailVerificationMessage,\n      EmailVerificationSubject: userPoolParams.EmailVerificationSubject,\n      VerificationMessageTemplate: userPoolParams.VerificationMessageTemplate,\n      SmsAuthenticationMessage: userPoolParams.SmsAuthenticationMessage,\n      MfaConfiguration: userPoolParams.MfaConfiguration,\n      DeviceConfiguration: userPoolParams.DeviceConfiguration,\n      SmsConfiguration: userPoolParams.SmsConfiguration,\n      UserPoolTags: userPoolParams.UserPoolTags,\n      UserPoolAddOns: userPoolParams.UserPoolAddOns,\n    };\n\n    // removing undefined keys\n    Object.keys(updateUserPoolConfig).forEach((key) => updateUserPoolConfig[key] === undefined && delete updateUserPoolConfig[key]);\n\n    /* removing UnusedAccountValidityDays as deprecated\n    InvalidParameterException: Please use TemporaryPasswordValidityDays in PasswordPolicy instead of UnusedAccountValidityDays\n    */\n    if (updateUserPoolConfig.AdminCreateUserConfig && updateUserPoolConfig.AdminCreateUserConfig.UnusedAccountValidityDays) {\n      delete updateUserPoolConfig.AdminCreateUserConfig.UnusedAccountValidityDays;\n    }\n    lambdaConfig.forEach((lambda) => (config[`${lambda.triggerType}`] = lambda.lambdaFunctionArn));\n    if (event.RequestType === 'Delete') {\n      try {\n        updateUserPoolConfig.LambdaConfig = {};\n        console.log(`${event.RequestType}:`, JSON.stringify(updateUserPoolConfig));\n        const result = await cognitoClient.updateUserPool(updateUserPoolConfig).promise();\n        console.log(`delete response data ${JSON.stringify(result)}`);\n        await response.send(event, context, response.SUCCESS, {}, physicalResourceId);\n      } catch (err) {\n        console.log(err.stack);\n        await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n      }\n    }\n    if (event.RequestType === 'Update' || event.RequestType === 'Create') {\n      updateUserPoolConfig.LambdaConfig = config;\n      try {\n        const result = await cognitoClient.updateUserPool(updateUserPoolConfig).promise();\n        console.log(`createOrUpdate response data ${JSON.stringify(result)}`);\n        await response.send(event, context, response.SUCCESS, {}, physicalResourceId);\n      } catch (err) {\n        console.log(err.stack);\n        await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n      }\n    }\n  } catch (err) {\n    console.log(err.stack);\n    await response.send(event, context, response.FAILED, { err }, physicalResourceId);\n  }\n};\n"
        },
        "Role": {
          "Fn::GetAtt": [
            "authTriggerFnServiceRole08093B67",
            "Arn"
          ]
        },
        "Handler": "index.handler",
        "Runtime": "nodejs14.x"
      },
      "DependsOn": [
        "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "authTriggerFnServiceRole08093B67"
      ]
    },
    "CustomAuthTriggerResource": {
      "Type": "Custom::CustomAuthTriggerResourceOutputs",
      "Properties": {
        "ServiceToken": {
          "Fn::GetAtt": [
            "authTriggerFn7FCFA449",
            "Arn"
          ]
        },
        "userpoolId": {
          "Ref": "userpoolId"
        },
        "lambdaConfig": [
          {
            "triggerType": "PreSignUp",
            "lambdaFunctionName": "myappwebappPreSignup",
            "lambdaFunctionArn": {
              "Ref": "functionmyappwebappPreSignupArn"
            }
          }
        ],
        "nonce": "cd55fb39-4ef8-416e-bc0b-178353ecc845"
      },
      "DependsOn": [
        "authTriggerFn7FCFA449",
        "authTriggerFnServiceRoleDefaultPolicyEC9285A8",
        "authTriggerFnServiceRole08093B67"
      ],
      "UpdateReplacePolicy": "Delete",
      "DeletionPolicy": "Delete"
    }
  }
}

After some changes, I think I accidentally deleted authTriggerFn7FCFA449 and the whole deployment stopped working. CloudFormation started complaining like this:

Function not found: arn:aws:lambda:eu-central-1:510148651032:function:amplify-amplify65170398e4384-authTriggerFn7FCFA449-wBSraCi16QuH
(Service: AWSLambda; Status Code: 404; Error Code: ResourceNotFoundException; Request ID: 7195cef0-164c-4095-9de3-8a8309086a99; Proxy: null)

After detecting the drift I managed to recreate the Lambda, with all the roles, tags, code, cfn-response properly set up. Now, there's no drift. Additionally, the previous error dissapears. However, CloudFormation waits for an hour and then is stuck at UPDATE_FAILED on CustomAuthTriggerResource with the following problem:

CloudFormation did not receive a response from your Custom Resource.
Please check your logs for requestId [44595a05-67c1-4594-b16b-096ce6506d96].
If you are using the Python cfn-response module, you may need to update your Lambda function code so that CloudFormation can attach the updated version.

All other resources (authTriggerFn7FCFA449, authTriggerFnServiceRole08093B67, authTriggerFnServiceRoleDefaultPolicyEC9285A8, UserPoolPreSignUpLambdaInvokePermission) are at CREATE_COMPLETE and UPDATE_COMPLETE.

Looking at the amplify-amplify65170398e4384-authTriggerFn7FCFA449-wBSraCi16QuH logs, there is no such request. How do I proceed from here?

No git pushes, amplify pushes or stack updates fix the situation.

Expected behavior

CloudFormation should be able to call authTriggerFn7FCFA449 and deploy my amplify website.

Reproduction steps

1. Add a sign up hook
2. Delete authTriggerFn7FCFA449
3. Fix drift and bring authTriggerFn7FCFA449 up
4. Try to deploy Amplify (via GitHub or CLI)

Project Identifier

a22f3ce8372073d5dc469c7b32da3a78

Log output

No response

Additional information

No response

Before submitting, please confirm:

• I have done my best to include a minimal, self-contained set of instructions for consistently reproducing the issue.
• I have removed any sensitive information from my code snippets and submission.

[fikisipi]

I think #9837 is related.

CustomAuthTriggerResource [...] As a consequence, user is unable to make any subsequent push using Amplify CLI.

Edit: Yes, the workaround fixes it. Hope Amplify works on this so we don't have to do this hack, ever.

1. Add one line showing the CloudFormation event at the beginning of the current Lambda function ...authTriggerFn7FCFA449 via Lambda console. For example, console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
2. Click the "continue update rollback" button on CloudFormation console as the AWS documentation shows
3. Watch the CloudWatch log stream of the above Lambda function, note the S3 pre-signed URL, PhysicalResourceId, StackId, LogicalResourceId. An output will look like

{
    "RequestType": "Update",
    "ServiceToken": "arn:aws:lambda:ap-southeast-2:1234567:function:amplify-test9656353151-dev-1-authTriggerFn7FCFA449-xyz",
    "ResponseURL": "https://cloudformation-custom-resource-response-apsoutheast2.s3-ap-southeast-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aap-sou...00d761385cd62013820dc780bc07d07e021f77",
    "StackId": "arn:aws:cloudformation:ap-southeast-2:1234567:stack/amplify-test9656353151-dev-165513-AuthTriggerCustomLambdaStack-VZC2CVEFA8GA/00876ee0-93a7-11ec-b1d1-xyz",
    "RequestId": "82bf2ca0-061e-4895-ad70-xyz",
    "LogicalResourceId": "CustomAuthTriggerResource",
    "PhysicalResourceId": "2022/02/22/[$LATEST]f72b84eb...54d7ef4",
    "ResourceType": "Custom::CustomAuthTriggerResourceOutputs",
    ...
}

1. According to this AWS documentation , manually construct a cURL SUCCESS request based on the observed information. Note that the cURL destination URL should be the the "ResponseURL" above. Since it is a custom resource, you cannot use CloudFormation CLI signal-resource to unblock it. The cURL request looks like:

$ curl -H 'Content-Type: ''' -X PUT -d '{"Status": "SUCCESS","PhysicalResourceId": "2022/02/22/[$LATEST]f72b84eb...54d7ef4","StackId": "arn:aws:cloudformation:ap-southeast-2:1234567:stack/amplify-test9656353151-dev-165513-AuthTriggerCustomLambdaStack-VZC2CVEFA8GA/00876ee0-93a7-11ec-b1d1-xyz","RequestId": "82bf2ca0-061e-4895-ad70-xyz","LogicalResourceId": "CustomAuthTriggerResource"}' 'https://cloudformation-custom-resource-response-apsoutheast2.s3-ap-southeast-2.amazonaws.com/arn%3Aaws%3Acloudformation%3Aap-sou...00d761385cd62013820dc780bc07d07e021f77'

1. Once you send the SUCCESS request out, CloudFormation stack will restore to UPDATE_ROLLBACK_COMPLETE state, and user can make further push.

Then adjust the lambda timeout in CF to 300 seconds.

[github-actions]

⚠️COMMENT VISIBILITY WARNING⚠️

Comments on closed issues are hard for our team to see.
If you need more assistance, please open a new issue that references this one.
If you wish to keep having a conversation with other community members under this issue feel free to do so.

[OperationalFallacy]

That's a pretty easy workaround.

I ponder if Amplify clean up this nonsense with inline custom resource Lambdas in gen2.