diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js index 8735e727c..2698d7983 100755 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js @@ -502,6 +502,17 @@ class DriftableStack extends cdk.Stack { } } +class EarlyValidationStack extends cdk.Stack { + constructor(parent, id, props) { + super(parent, id, props); + + new s3.Bucket(this, 'MyBucket', { + bucketName: process.env.BUCKET_NAME, + removalPolicy: cdk.RemovalPolicy.DESTROY, + }); + } +} + class IamRolesStack extends cdk.Stack { constructor(parent, id, props) { super(parent, id, props); @@ -971,6 +982,9 @@ switch (stackSet) { new MetadataStack(app, `${stackPrefix}-metadata`); new DriftableStack(app, `${stackPrefix}-driftable`); + + new EarlyValidationStack(app, `${stackPrefix}-early-validation-stack1`); + new EarlyValidationStack(app, `${stackPrefix}-early-validation-stack2`); break; case 'stage-using-context': diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-early-validation.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-early-validation.integtest.ts new file mode 100644 index 000000000..e290a2631 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-early-validation.integtest.ts @@ -0,0 +1,32 @@ +import { randomUUID } from 'node:crypto'; +import { integTest, withDefaultFixture } from '../../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +integTest( + 'deploy - early validation error', + withDefaultFixture(async (fixture) => { + const bucketName = randomUUID(); + + // First, deploy a stack that creates a bucket with a custom name, + // which we expect to succeed + await fixture.cdkDeploy('early-validation-stack1', { + modEnv: { + BUCKET_NAME: bucketName, + }, + }); + + // Then deploy a different instance of the stack, that creates another + // bucket with the same name, to induce an early validation error + const stdErr = await fixture.cdkDeploy('early-validation-stack2', { + modEnv: { + BUCKET_NAME: bucketName, + }, + allowErrExit: true, + }); + + expect(stdErr).toContain(`Resource of type 'AWS::S3::Bucket' with identifier '${bucketName}' already exists`, + ); + }), +); + diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts b/packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts index 87c8c6a14..82583fe2b 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts @@ -100,9 +100,14 @@ import type { DescribeStackResourceDriftsCommandInput, ExecuteStackRefactorCommandInput, DescribeStackRefactorCommandInput, - CreateStackRefactorCommandOutput, ExecuteStackRefactorCommandOutput, + CreateStackRefactorCommandOutput, + ExecuteStackRefactorCommandOutput, + DescribeEventsCommandOutput, + DescribeEventsCommandInput, } from '@aws-sdk/client-cloudformation'; import { + paginateDescribeEvents, + paginateListStacks, CloudFormationClient, ContinueUpdateRollbackCommand, @@ -113,6 +118,7 @@ import { DeleteChangeSetCommand, DeleteGeneratedTemplateCommand, DeleteStackCommand, + DescribeEventsCommand, DescribeChangeSetCommand, DescribeGeneratedTemplateCommand, DescribeResourceScanCommand, @@ -141,6 +147,7 @@ import { waitUntilStackRefactorCreateComplete, waitUntilStackRefactorExecuteComplete, } from '@aws-sdk/client-cloudformation'; +import type { OperationEvent } from '@aws-sdk/client-cloudformation/dist-types/models/models_0'; import type { FilterLogEventsCommandInput, FilterLogEventsCommandOutput, @@ -434,6 +441,7 @@ export interface ICloudFormationClient { deleteChangeSet(input: DeleteChangeSetCommandInput): Promise; deleteGeneratedTemplate(input: DeleteGeneratedTemplateCommandInput): Promise; deleteStack(input: DeleteStackCommandInput): Promise; + describeEvents(input: DescribeEventsCommandInput): Promise; describeChangeSet(input: DescribeChangeSetCommandInput): Promise; describeGeneratedTemplate( input: DescribeGeneratedTemplateCommandInput, @@ -468,6 +476,7 @@ export interface ICloudFormationClient { describeStackEvents(input: DescribeStackEventsCommandInput): Promise; listStackResources(input: ListStackResourcesCommandInput): Promise; paginatedListStacks(input: ListStacksCommandInput): Promise; + paginatedDescribeEvents(input: DescribeEventsCommandInput): Promise; createStackRefactor(input: CreateStackRefactorCommandInput): Promise; executeStackRefactor(input: ExecuteStackRefactorCommandInput): Promise; waitUntilStackRefactorCreateComplete(input: DescribeStackRefactorCommandInput): Promise; @@ -710,6 +719,8 @@ export class SDK { client.send(new DetectStackDriftCommand(input)), detectStackResourceDrift: (input: DetectStackResourceDriftCommandInput): Promise => client.send(new DetectStackResourceDriftCommand(input)), + describeEvents: (input: DescribeEventsCommandInput): Promise => + client.send(new DescribeEventsCommand(input)), describeChangeSet: (input: DescribeChangeSetCommandInput): Promise => client.send(new DescribeChangeSetCommand(input)), describeGeneratedTemplate: ( @@ -775,6 +786,14 @@ export class SDK { } return stackResources; }, + paginatedDescribeEvents: async (input: DescribeEventsCommandInput): Promise => { + const stackResources = Array(); + const paginator = paginateDescribeEvents({ client }, input); + for await (const page of paginator) { + stackResources.push(...(page.OperationEvents || [])); + } + return stackResources; + }, createStackRefactor: (input: CreateStackRefactorCommandInput): Promise => { return client.send(new CreateStackRefactorCommand(input)); }, diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts index 0abbdb09a..def3a71c5 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/cfn-api.ts @@ -21,6 +21,10 @@ import { CloudFormationStack, makeBodyParameter } from '../cloudformation'; import type { IoHelper } from '../io/private'; import type { ResourcesToImport } from '../resource-import'; +export interface ValidationReporter { + report(changeSetName: string, stackName: string): Promise; +} + /** * Describe a changeset in CloudFormation, regardless of its current state. * @@ -103,7 +107,7 @@ export async function waitForChangeSet( ioHelper: IoHelper, stackName: string, changeSetName: string, - { fetchAll }: { fetchAll: boolean }, + { fetchAll, validationReporter }: { fetchAll: boolean; validationReporter?: ValidationReporter }, ): Promise { await ioHelper.defaults.debug(format('Waiting for changeset %s on stack %s to finish creating...', changeSetName, stackName)); const ret = await waitFor(async () => { @@ -121,10 +125,17 @@ export async function waitForChangeSet( return description; } - // eslint-disable-next-line @stylistic/max-len - throw new ToolkitError( - `Failed to create ChangeSet ${changeSetName} on ${stackName}: ${description.Status || 'NO_STATUS'}, ${description.StatusReason || 'no reason provided'}`, - ); + if (description.Status === ChangeSetStatus.FAILED && description.StatusReason?.includes('AWS::EarlyValidation')) { + await validationReporter?.report(changeSetName, stackName); + return description; + } else { + // eslint-disable-next-line @stylistic/max-len + throw new ToolkitError( + `Failed to create ChangeSet ${changeSetName} on ${stackName}: ${description.Status || 'NO_STATUS'}, ${ + description.StatusReason || 'no reason provided' + }`, + ); + } }); if (!ret) { diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts index 189a9f965..c37dc6510 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/deploy-stack.ts @@ -38,6 +38,7 @@ import { tryHotswapDeployment } from '../hotswap/hotswap-deployments'; import type { IoHelper } from '../io/private'; import type { ResourcesToImport } from '../resource-import'; import { StackActivityMonitor } from '../stack-events'; +import { EarlyValidationReporter } from './early-validation'; export interface DeployStackOptions { /** @@ -511,8 +512,10 @@ class FullCloudFormationDeployment { await this.ioHelper.defaults.debug(format('Initiated creation of changeset: %s; waiting for it to finish creating...', changeSet.Id)); // Fetching all pages if we'll execute, so we can have the correct change count when monitoring. + const validationReporter = new EarlyValidationReporter(this.options.sdk, this.ioHelper); return waitForChangeSet(this.cfn, this.ioHelper, this.stackName, changeSetName, { fetchAll: willExecute, + validationReporter, }); } diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/deployments/early-validation.ts b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/early-validation.ts new file mode 100644 index 000000000..30b29d430 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/lib/api/deployments/early-validation.ts @@ -0,0 +1,49 @@ +import type { OperationEvent } from '@aws-sdk/client-cloudformation'; +import type { ValidationReporter } from './cfn-api'; +import { ToolkitError } from '../../toolkit/toolkit-error'; +import type { SDK } from '../aws-auth/sdk'; +import type { IoHelper } from '../io/private'; + +/** + * A ValidationReporter that checks for early validation errors right after + * creating the change set. If any are found, it throws an error listing all validation failures. + * If the DescribeEvents API call fails (for example, due to insufficient permissions), + * it logs a warning instead. + */ +export class EarlyValidationReporter implements ValidationReporter { + constructor(private readonly sdk: SDK, private readonly ioHelper: IoHelper) { + } + + public async report(changeSetName: string, stackName: string) { + let operationEvents: OperationEvent[] = []; + try { + operationEvents = await this.getFailedEvents(stackName, changeSetName); + } catch (error) { + const message = + 'While creating the change set, CloudFormation detected errors in the generated templates,' + + ' but the deployment role does not have permissions to call the DescribeEvents API to retrieve details about these errors.\n' + + 'To see more details, re-bootstrap your environment, or otherwise ensure that the deployment role has permissions to call the DescribeEvents API.'; + + await this.ioHelper.defaults.warn(message); + } + + if (operationEvents.length > 0) { + const failures = operationEvents + .map((event) => ` - ${event.ValidationStatusReason} (at ${event.ValidationPath})`) + .join('\n'); + + const message = `ChangeSet '${changeSetName}' on stack '${stackName}' failed early validation:\n${failures}`; + throw new ToolkitError(message); + } + } + + private async getFailedEvents(stackName: string, changeSetName: string) { + return this.sdk.cloudFormation().paginatedDescribeEvents({ + StackName: stackName, + ChangeSetName: changeSetName, + Filters: { + FailedEvents: true, + }, + }); + } +} diff --git a/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts index ef9291b60..ed0792e8c 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/deployments/deploy-stack.test.ts @@ -7,6 +7,7 @@ import { DeleteChangeSetCommand, DeleteStackCommand, DescribeChangeSetCommand, + DescribeEventsCommand, DescribeStacksCommand, ExecuteChangeSetCommand, type ExecuteChangeSetCommandInput, @@ -34,6 +35,7 @@ import { TestIoHost } from '../../_helpers/test-io-host'; let ioHost = new TestIoHost(); let ioHelper = ioHost.asHelper('deploy'); +let ioHelperWarn: jest.SpyInstance, [input: string, ...args: unknown[]], any>; function testDeployStack(options: DeployStackApiOptions) { return deployStack(options, ioHelper); @@ -111,6 +113,7 @@ beforeEach(() => { mockCloudFormationClient.on(UpdateTerminationProtectionCommand).resolves({ StackId: 'stack-id', }); + ioHelperWarn = jest.spyOn(ioHelper.defaults, 'warn'); }); function standardDeployStackArguments(): DeployStackApiOptions { @@ -763,6 +766,49 @@ test('deployStack reports no change if describeChangeSet returns specific error' expect(deployResult.type === 'did-deploy-stack' && deployResult.noOp).toEqual(true); }); +test('deployStack throws error in case of early validation failures', async () => { + mockCloudFormationClient.on(DescribeChangeSetCommand).resolvesOnce({ + Status: ChangeSetStatus.FAILED, + StatusReason: '(AWS::EarlyValidation::SomeError). Blah blah blah.', + }); + + mockCloudFormationClient.on(DescribeEventsCommand).resolves({ + OperationEvents: [ + { + ValidationStatus: 'FAILED', + ValidationStatusReason: 'Resource already exists', + ValidationPath: 'Resources/MyResource', + }, + ], + }); + + await expect( + testDeployStack({ + ...standardDeployStackArguments(), + }), + ).rejects.toThrow(`ChangeSet 'cdk-deploy-change-set' on stack 'withouterrors' failed early validation: + - Resource already exists (at Resources/MyResource)`); +}); + +test('deployStack warns when it cannot get the events in case of early validation errors', async () => { + mockCloudFormationClient.on(DescribeChangeSetCommand).resolvesOnce({ + Status: ChangeSetStatus.FAILED, + StatusReason: '(AWS::EarlyValidation::SomeError). Blah blah blah.', + }); + + mockCloudFormationClient.on(DescribeEventsCommand).rejectsOnce({ + message: 'AccessDenied', + }); + + await testDeployStack({ + ...standardDeployStackArguments(), + }); + + expect(ioHelperWarn).toHaveBeenCalledWith( + expect.stringContaining('does not have permissions to call the DescribeEvents API'), + ); +}); + test('deploy not skipped if template did not change but one tag removed', async () => { // GIVEN mockCloudFormationClient.on(DescribeStacksCommand).resolves({ diff --git a/packages/@aws-cdk/toolkit-lib/test/api/deployments/early-validation.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/deployments/early-validation.test.ts new file mode 100644 index 000000000..7e22c51a5 --- /dev/null +++ b/packages/@aws-cdk/toolkit-lib/test/api/deployments/early-validation.test.ts @@ -0,0 +1,46 @@ +import { EarlyValidationReporter } from '../../../lib/api/deployments/early-validation'; + +it('throws an error when there are failed validation events', async () => { + const sdkMock = { + cloudFormation: jest.fn().mockReturnValue({ + paginatedDescribeEvents: jest.fn().mockResolvedValue([ + { ValidationStatusReason: 'Resource already exists', ValidationPath: 'Resources/MyResource' }, + ]), + }), + }; + const ioHelperMock = { defaults: { warn: jest.fn() } }; + const reporter = new EarlyValidationReporter(sdkMock as any, ioHelperMock as any); + + await expect(reporter.report('test-change-set', 'test-stack')).rejects.toThrow( + "ChangeSet 'test-change-set' on stack 'test-stack' failed early validation:\n - Resource already exists (at Resources/MyResource)", + ); +}); + +it('does not throw when there are no failed validation events', async () => { + const sdkMock = { + cloudFormation: jest.fn().mockReturnValue({ + paginatedDescribeEvents: jest.fn().mockResolvedValue([]), + }), + }; + const ioHelperMock = { defaults: { warn: jest.fn() } }; + const reporter = new EarlyValidationReporter(sdkMock as any, ioHelperMock as any); + + await expect(reporter.report('test-change-set', 'test-stack')).resolves.not.toThrow(); + expect(ioHelperMock.defaults.warn).not.toHaveBeenCalled(); +}); + +it('logs a warning when DescribeEvents API call fails', async () => { + const sdkMock = { + cloudFormation: jest.fn().mockReturnValue({ + paginatedDescribeEvents: jest.fn().mockRejectedValue(new Error('AccessDenied')), + }), + }; + const ioHelperMock = { defaults: { warn: jest.fn() } }; + const reporter = new EarlyValidationReporter(sdkMock as any, ioHelperMock as any); + + await reporter.report('test-change-set', 'test-stack'); + + expect(ioHelperMock.defaults.warn).toHaveBeenCalledWith( + expect.stringContaining('While creating the change set, CloudFormation detected errors in the generated templates'), + ); +}); diff --git a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml index 5f98cdad8..32c4f8521 100644 --- a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml +++ b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml @@ -631,6 +631,7 @@ Resources: - cloudformation:DeleteChangeSet - cloudformation:DescribeChangeSet - cloudformation:DescribeStacks + - cloudformation:DescribeEvents - cloudformation:ExecuteChangeSet - cloudformation:CreateStack - cloudformation:UpdateStack @@ -814,7 +815,7 @@ Resources: Name: Fn::Sub: '/cdk-bootstrap/${Qualifier}/version' # Also update this value below (see comment there) - Value: '29' + Value: '30' Outputs: BucketName: Description: The name of the S3 bucket owned by the CDK toolkit stack @@ -849,4 +850,4 @@ Outputs: # {Fn::GetAtt} on an SSM Parameter is eventually consistent, and can fail with "parameter # doesn't exist" even after just having been created. To reduce our deploy failure rate, we # duplicate the value here and use a build-time test to ensure the two values are the same. - Value: '29' + Value: '30'