feat: Fetch SNS topic ARN from SQL manifest (#2345)

packages/amplify-category-api/src/graphql-transformer/transform-graphql-schema-v2.ts

@@ -16,6 +16,7 @@ import {
   ModelDataSourceStrategyDbType,
   ModelDataSourceStrategySqlDbType,
   RDSLayerMapping,
+  RDSSNSTopicMapping,
   SQLLambdaModelDataSourceStrategy,
   SqlDirectiveDataSourceStrategy,
   SqlModelDataSourceDbConnectionConfig,
@@ -343,8 +344,10 @@ const buildAPIProject = async (context: $TSContext, opts: TransformerProjectOpti
 
   // Read the RDS Mapping S3 Manifest only if the schema contains SQL models or @sql directives.
   let rdsLayerMapping: RDSLayerMapping | undefined = undefined;
+  let rdsSnsTopicMapping: RDSSNSTopicMapping | undefined = undefined;
   if (containsSqlModelOrDirective(dataSourceStrategies, sqlDirectiveDataSourceStrategies)) {
     rdsLayerMapping = await getRDSLayerMapping(context, useBetaSqlLayer);
+    rdsSnsTopicMapping = await getRDSSNSTopicMapping(context, useBetaSqlLayer);
   }
 
   const transformManager = new TransformManager(
@@ -367,6 +370,7 @@ const buildAPIProject = async (context: $TSContext, opts: TransformerProjectOpti
     sqlDirectiveDataSourceStrategies,
     printTransformerLog,
     rdsLayerMapping,
+    rdsSnsTopicMapping,
   });
 
   const transformOutput: DeploymentResources = {
@@ -401,7 +405,7 @@ export const getUserOverridenSlots = (userDefinedSlots: Record<string, UserDefin
     .filter((slotName) => slotName !== undefined);
 
 const getRDSLayerMapping = async (context: $TSContext, useBetaSqlLayer = false): Promise<RDSLayerMapping> => {
-  const bucket = `${ResourceConstants.RESOURCES.SQLLayerVersionManifestBucket}${useBetaSqlLayer ? '-beta' : ''}`;
+  const bucket = `${ResourceConstants.RESOURCES.SQLLayerManifestBucket}${useBetaSqlLayer ? '-beta' : ''}`;
   const region = context.amplify.getProjectMeta().providers.awscloudformation.Region;
   const url = `https://${bucket}.s3.amazonaws.com/${ResourceConstants.RESOURCES.SQLLayerVersionManifestKeyPrefix}${region}`;
   const response = await fetch(url);
@@ -418,6 +422,24 @@ const getRDSLayerMapping = async (context: $TSContext, useBetaSqlLayer = false):
   }
 };
 
+const getRDSSNSTopicMapping = async (context: $TSContext, useBetaSqlLayer = false): Promise<RDSSNSTopicMapping> => {
+  const bucket = `${ResourceConstants.RESOURCES.SQLLayerManifestBucket}${useBetaSqlLayer ? '-beta' : ''}`;
+  const region = context.amplify.getProjectMeta().providers.awscloudformation.Region;
+  const url = `https://${bucket}.s3.amazonaws.com/${ResourceConstants.RESOURCES.SQLSNSTopicARNManifestKeyPrefix}${region}`;
+  const response = await fetch(url);
+  if (response.status === 200) {
+    const result = await response.text();
+    const mapping = {
+      [region]: {
+        topicArn: result,
+      },
+    };
+    return mapping as RDSSNSTopicMapping;
+  } else {
+    throw new Error(`Unable to retrieve sns topic ARN mapping from ${url} with status code ${response.status}.`);
+  }
+};
+
 const isSqlLambdaVpcConfigRequired = async (context: $TSContext, dbType: ModelDataSourceStrategyDbType): Promise<VpcConfig | undefined> => {
   // If the database is in VPC, we will use the same VPC configuration for the SQL lambda.
   // Customers are required to add inbound rule for port 443 from the private subnet in the Security Group.

packages/amplify-e2e-tests/src/rds-v2-tests-common/rds-model-v2.ts

@@ -18,7 +18,6 @@ import { existsSync, readFileSync } from 'fs-extra';
 import generator from 'generate-password';
 import { ObjectTypeDefinitionNode, parse } from 'graphql';
 import AWSAppSyncClient, { AUTH_TYPE } from 'aws-appsync';
-import { ResourceConstants } from 'graphql-transformer-common';
 import gql from 'graphql-tag';
 import {
   ImportedRDSType,
@@ -37,7 +36,6 @@ export const testRDSModel = (engine: ImportedRDSType, queries: string[]): void =
   const CDK_VPC_ENDPOINT_TYPE = 'AWS::EC2::VPCEndpoint';
   const CDK_SUBSCRIPTION_TYPE = 'AWS::SNS::Subscription';
   const APPSYNC_DATA_SOURCE_TYPE = 'AWS::AppSync::DataSource';
-  const { AmplifySQLLayerNotificationTopicAccount, AmplifySQLLayerNotificationTopicName } = ResourceConstants.RESOURCES;
 
   describe(`RDS Model Directive - ${engine}`, () => {
     const [db_user, db_password, db_identifier] = generator.generateMultiple(3);
@@ -138,13 +136,6 @@ export const testRDSModel = (engine: ImportedRDSType, queries: string[]): void =
       );
 
       // Validate subscription
-      const expectedTopicArn = {
-        'Fn::Join': [
-          ':',
-          ['arn:aws:sns', { Ref: 'AWS::Region' }, `${AmplifySQLLayerNotificationTopicAccount}:${AmplifySQLLayerNotificationTopicName}`],
-        ],
-      };
-
       // Counterintuitively, the subscription actually gets created with the resource prefix of the FUNCTION that gets triggered,
       // rather than the scope created specifically for the subscription
       const rdsPatchingSubscription = getResource(resources, resourceNames.sqlPatchingLambdaFunction, CDK_SUBSCRIPTION_TYPE);
@@ -154,7 +145,10 @@ export const testRDSModel = (engine: ImportedRDSType, queries: string[]): void =
       expect(rdsPatchingSubscription.Properties.Protocol).toEqual('lambda');
       expect(rdsPatchingSubscription.Properties.Endpoint).toBeDefined();
       expect(rdsPatchingSubscription.Properties.TopicArn).toBeDefined();
-      expect(rdsPatchingSubscription.Properties.TopicArn).toMatchObject(expectedTopicArn);
+      const topicArnMappingRef = rdsPatchingSubscription.Properties.TopicArn['Fn::FindInMap'];
+      expect(topicArnMappingRef?.length).toEqual(3);
+      expect(topicArnMappingRef[1]).toEqual({ Ref: 'AWS::Region' });
+      expect(topicArnMappingRef[2]).toEqual('topicArn');
       expect(rdsPatchingSubscription.Properties.Region).toBeDefined();
       expect(rdsPatchingSubscription.Properties.FilterPolicy).toBeDefined();
       expect(rdsPatchingSubscription.Properties.FilterPolicy.Region).toBeDefined();

packages/amplify-graphql-api-construct/.jsii

@@ -3998,7 +3998,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 285
+            "line": 286
           },
           "name": "addDynamoDbDataSource",
           "parameters": [
@@ -4047,7 +4047,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 297
+            "line": 298
           },
           "name": "addElasticsearchDataSource",
           "parameters": [
@@ -4094,7 +4094,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 307
+            "line": 308
           },
           "name": "addEventBridgeDataSource",
           "parameters": [
@@ -4141,7 +4141,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 389
+            "line": 390
           },
           "name": "addFunction",
           "parameters": [
@@ -4176,7 +4176,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 318
+            "line": 319
           },
           "name": "addHttpDataSource",
           "parameters": [
@@ -4224,7 +4224,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 329
+            "line": 330
           },
           "name": "addLambdaDataSource",
           "parameters": [
@@ -4272,7 +4272,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 340
+            "line": 341
           },
           "name": "addNoneDataSource",
           "parameters": [
@@ -4311,7 +4311,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 351
+            "line": 352
           },
           "name": "addOpenSearchDataSource",
           "parameters": [
@@ -4359,7 +4359,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 364
+            "line": 365
           },
           "name": "addRdsDataSource",
           "parameters": [
@@ -4426,7 +4426,7 @@
           },
           "locationInModule": {
             "filename": "src/amplify-graphql-api.ts",
-            "line": 380
+            "line": 381
           },
           "name": "addResolver",
           "parameters": [

packages/amplify-graphql-api-construct/src/amplify-graphql-api.ts

@@ -218,6 +218,7 @@ export class AmplifyGraphqlApi extends Construct {
       // CDK construct uses a custom resource. We'll define this explicitly here to remind ourselves that this value is unused in the CDK
       // construct flow
       rdsLayerMapping: undefined,
+      rdsSnsTopicMapping: undefined,
       ...getDataSourceStrategiesProvider(definition),
     };
 

packages/amplify-graphql-model-transformer/src/resolvers/rds/resolver.ts

@@ -29,6 +29,7 @@ import {
   isSqlModelDataSourceSsmDbConnectionConfig,
   isSqlModelDataSourceSecretsManagerDbConnectionConfig,
   isSqlModelDataSourceSsmDbConnectionStringConfig,
+  RDSSNSTopicMapping,
 } from '@aws-amplify/graphql-transformer-interfaces';
 import { Effect, IRole, Policy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
 import { IFunction, LayerVersion, Runtime, Alias, Function as LambdaFunction } from 'aws-cdk-lib/aws-lambda';
@@ -65,6 +66,18 @@ export const setRDSLayerMappings = (scope: Construct, mapping: RDSLayerMapping,
     mapping,
   });
 
+/**
+ * Define RDS Patching SNS Topic ARN region mappings. The optional `mapping` can be specified in place of the defaults that are hardcoded at the time
+ * this package is published. For the CLI flow, the `mapping` will be downloaded at runtime during the `amplify push` flow. For the CDK,
+ * the layer version will be resolved by a custom CDK resource.
+ * @param scope Construct
+ * @param mapping an RDSSNSTopicMapping to use in place of the defaults
+ */
+export const setRDSSNSTopicMappings = (scope: Construct, mapping: RDSSNSTopicMapping, resourceNames: SQLLambdaResourceNames): CfnMapping =>
+  new CfnMapping(scope, resourceNames.sqlSNSTopicArnMapping, {
+    mapping,
+  });
+
 /**
  * Create RDS Lambda function
  * @param scope Construct
@@ -158,22 +171,55 @@ export const createRdsLambda = (
  * because it would have no effect.
  */
 export const createLayerVersionCustomResource = (scope: Construct, resourceNames: SQLLambdaResourceNames): AwsCustomResource => {
-  const { SQLLayerVersionManifestBucket, SQLLayerVersionManifestBucketRegion, SQLLayerVersionManifestKeyPrefix } =
-    ResourceConstants.RESOURCES;
+  const { SQLLayerManifestBucket, SQLLayerManifestBucketRegion, SQLLayerVersionManifestKeyPrefix } = ResourceConstants.RESOURCES;
 
   const key = Fn.join('', [SQLLayerVersionManifestKeyPrefix, Fn.ref('AWS::Region')]);
 
-  const manifestArn = `arn:aws:s3:::${SQLLayerVersionManifestBucket}/${key}`;
+  const manifestArn = `arn:aws:s3:::${SQLLayerManifestBucket}/${key}`;
 
   const resourceName = resourceNames.sqlLayerVersionResolverCustomResource;
   const customResource = new AwsCustomResource(scope, resourceName, {
     resourceType: 'Custom::SQLLayerVersionCustomResource',
     onUpdate: {
       service: 'S3',
       action: 'getObject',
-      region: SQLLayerVersionManifestBucketRegion,
+      region: SQLLayerManifestBucketRegion,
+      parameters: {
+        Bucket: SQLLayerManifestBucket,
+        Key: key,
+      },
+      // Make the physical ID change each time we do a deployment, so we always check for the latest version. This means we will never have
+      // a strictly no-op deployment, but the SQL Lambda configuration won't change unless the actual layer value changes
+      physicalResourceId: PhysicalResourceId.of(`${resourceName}-${Date.now().toString()}`),
+    },
+    policy: AwsCustomResourcePolicy.fromSdkCalls({
+      resources: [manifestArn],
+    }),
+  });
+
+  return customResource;
+};
+
+/**
+ * Generates an AwsCustomResource to resolve the SNS Topic ARNs that the lambda used for updating the SQL Lambda Layer version installed into the customer
+ * account.
+ */
+export const createSNSTopicARNCustomResource = (scope: Construct, resourceNames: SQLLambdaResourceNames): AwsCustomResource => {
+  const { SQLLayerManifestBucket, SQLLayerManifestBucketRegion, SQLSNSTopicARNManifestKeyPrefix } = ResourceConstants.RESOURCES;
+
+  const key = Fn.join('', [SQLSNSTopicARNManifestKeyPrefix, Fn.ref('AWS::Region')]);
+
+  const manifestArn = `arn:aws:s3:::${SQLLayerManifestBucket}/${key}`;
+
+  const resourceName = resourceNames.sqlSNSTopicARNResolverCustomResource;
+  const customResource = new AwsCustomResource(scope, resourceName, {
+    resourceType: 'Custom::SQLSNSTopicARNCustomResource',
+    onUpdate: {
+      service: 'S3',
+      action: 'getObject',
+      region: SQLLayerManifestBucketRegion,
       parameters: {
-        Bucket: SQLLayerVersionManifestBucket,
+        Bucket: SQLLayerManifestBucket,
         Key: key,
       },
       // Make the physical ID change each time we do a deployment, so we always check for the latest version. This means we will never have

packages/amplify-graphql-model-transformer/src/resources/rds-model-resource-generator.ts

@@ -16,7 +16,6 @@ import {
   isSqlModelDataSourceSecretsManagerDbConnectionConfig,
   isSqlModelDataSourceSsmDbConnectionStringConfig,
 } from '@aws-amplify/graphql-transformer-interfaces';
-import { ResourceConstants } from 'graphql-transformer-common';
 import { LambdaDataSource } from 'aws-cdk-lib/aws-appsync';
 import { ObjectTypeDefinitionNode } from 'graphql';
 import { ModelVTLGenerator, RDSModelVTLGenerator } from '../resolvers';
@@ -27,7 +26,9 @@ import {
   createRdsPatchingLambda,
   createRdsPatchingLambdaRole,
   setRDSLayerMappings,
+  setRDSSNSTopicMappings,
   CredentialStorageMethod,
+  createSNSTopicARNCustomResource,
 } from '../resolvers/rds';
 import { ModelResourceGenerator } from './model-resource-generator';
 
@@ -102,8 +103,6 @@ export class RdsModelResourceGenerator extends ModelResourceGenerator {
     const engine = getImportedRDSTypeFromStrategyDbType(dbType);
     const dbConnectionConfig = strategy.dbConnectionConfig;
     const secretEntry = strategy.dbConnectionConfig;
-    const { AmplifySQLLayerNotificationTopicAccount, AmplifySQLLayerNotificationTopicName } = ResourceConstants.RESOURCES;
-
     const lambdaRoleScope = context.stackManager.getScopeFor(resourceNames.sqlLambdaExecutionRole, resourceNames.sqlStack);
     const lambdaScope = context.stackManager.getScopeFor(resourceNames.sqlLambdaFunction, resourceNames.sqlStack);
 
@@ -167,14 +166,7 @@ export class RdsModelResourceGenerator extends ModelResourceGenerator {
     });
 
     // Add SNS subscription for patching notifications
-    const topicArn = Fn.join(':', [
-      'arn',
-      'aws',
-      'sns',
-      Fn.ref('AWS::Region'),
-      AmplifySQLLayerNotificationTopicAccount,
-      AmplifySQLLayerNotificationTopicName,
-    ]);
+    const topicArn = resolveSNSTopicARN(lambdaScope, context, resourceNames);
 
     const patchingSubscriptionScope = context.stackManager.getScopeFor(resourceNames.sqlPatchingSubscription, resourceNames.sqlStack);
     const snsTopic = Topic.fromTopicArn(patchingSubscriptionScope, resourceNames.sqlPatchingTopic, topicArn);
@@ -244,3 +236,24 @@ const resolveLayerVersion = (scope: Construct, context: TransformerContextProvid
   }
   return layerVersionArn;
 };
+
+/**
+ * Resolves the SNS topic ARN that the patching lambda in the customer's account subscribes to listen for lambda layer updates from the service. In the Gen1 CLI flow, the transform-graphql-schema-v2
+ * buildAPIProject function retrieves the latest layer version from the S3 bucket. In the CDK construct, such async behavior at synth time
+ * is forbidden, so we use an AwsCustomResource to resolve the latest layer version. The AwsCustomResource does not work with the CLI custom
+ * synth functionality, so we fork the behavior at this point.
+ *
+ * Note that in either case, the returned value is not actually the literal layer ARN, but rather a reference to be resolved at deploy time:
+ * in the CLI case, it's the resolution of the SQLLayerMapping; in the CDK case, it's the 'Body' response field from the AwsCustomResource's
+ * invocation of s3::GetObject.
+ *
+ * TODO: Remove this once we remove SQL imports from Gen1 CLI.
+ */
+const resolveSNSTopicARN = (scope: Construct, context: TransformerContextProvider, resourceNames: SQLLambdaResourceNames): string => {
+  if (context.rdsSnsTopicMapping) {
+    setRDSSNSTopicMappings(scope, context.rdsSnsTopicMapping, resourceNames);
+    return Fn.findInMap(resourceNames.sqlSNSTopicArnMapping, Fn.ref('AWS::Region'), 'topicArn');
+  }
+  const layerVersionCustomResource = createSNSTopicARNCustomResource(scope, resourceNames);
+  return layerVersionCustomResource.getResponseField('Body');
+};

packages/amplify-graphql-transformer-core/API.md

@@ -59,6 +59,8 @@ import { OperationTypeDefinitionNode } from 'graphql';
 import { QueryFieldType } from '@aws-amplify/graphql-transformer-interfaces';
 import { RDSLayerMapping } from '@aws-amplify/graphql-transformer-interfaces';
 import { RDSLayerMappingProvider } from '@aws-amplify/graphql-transformer-interfaces';
+import { RDSSNSTopicMapping } from '@aws-amplify/graphql-transformer-interfaces';
+import { RDSSNSTopicMappingProvider } from '@aws-amplify/graphql-transformer-interfaces';
 import { S3MappingTemplateProvider } from '@aws-amplify/graphql-transformer-interfaces';
 import { SchemaDefinitionNode } from 'graphql';
 import { SqlDirectiveDataSourceStrategy } from '@aws-amplify/graphql-transformer-interfaces';
@@ -279,7 +281,7 @@ export class GraphQLTransform {
     // Warning: (ae-forgotten-export) The symbol "TransformOption" needs to be exported by the entry point index.d.ts
     //
     // (undocumented)
-    transform({ assetProvider, dataSourceStrategies, nestedStackProvider, parameterProvider, rdsLayerMapping, schema, scope, sqlDirectiveDataSourceStrategies, synthParameters, }: TransformOption): void;
+    transform({ assetProvider, dataSourceStrategies, nestedStackProvider, parameterProvider, rdsLayerMapping, rdsSnsTopicMapping, schema, scope, sqlDirectiveDataSourceStrategies, synthParameters, }: TransformOption): void;
 }
 
 // @public (undocumented)
@@ -552,6 +554,10 @@ export interface SQLLambdaResourceNames {
     // (undocumented)
     sqlPatchingTopic: string;
     // (undocumented)
+    sqlSNSTopicArnMapping: string;
+    // (undocumented)
+    sqlSNSTopicARNResolverCustomResource: string;
+    // (undocumented)
     sqlStack: string;
     // (undocumented)
     sqlVpcEndpointPrefix: string;