From ac79325dcadd37f04b017f6b5e86328edf3d9194 Mon Sep 17 00:00:00 2001 From: Tom Wright Date: Thu, 11 May 2023 14:37:13 +1000 Subject: [PATCH] docs: add documentation --- README.md | 369 +++++++++++++++++++++++++++++++--- bin/appmesh.ts | 4 +- bin/bluegreen.ts | 14 +- bin/serviceconnect.ts | 3 +- docs/appmesh-components.svg | 3 + docs/appmesh.svg | 3 + docs/bluegreen.svg | 3 + docs/serviceconnect.svg | 3 + lib/appmesh.ts | 61 +++++- lib/appmesh/cluster.ts | 17 ++ lib/appmesh/express.ts | 54 ++++- lib/appmesh/mesh.ts | 38 ++++ lib/bluegreen.ts | 26 ++- lib/bluegreen/deployment.ts | 87 +++++++- lib/ip.ts | 3 + lib/serviceconnect.ts | 19 ++ lib/serviceconnect/cluster.ts | 7 + lib/serviceconnect/express.ts | 24 +++ 18 files changed, 687 insertions(+), 51 deletions(-) create mode 100644 docs/appmesh-components.svg create mode 100644 docs/appmesh.svg create mode 100644 docs/bluegreen.svg create mode 100644 docs/serviceconnect.svg create mode 100644 lib/ip.ts diff --git a/README.md b/README.md index 03400bc..b0cc558 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,357 @@ -# Welcome to your CDK TypeScript project +# ECS Service Connect and App Mesh -This is a blank project for CDK development with TypeScript. +This repository explores the experience of using the AWS CDK to administer ECS Service Connect and App Mesh. -The `cdk.json` file tells the CDK Toolkit how to execute your app. +## Setup -## Useful commands +Install dependencies -* `npm run build` compile typescript to js -* `npm run watch` watch for changes and compile -* `npm run test` perform the jest unit tests -* `cdk deploy` deploy this stack to your default AWS account/region -* `cdk diff` compare deployed stack with current state -* `cdk synth` emits the synthesized CloudFormation template +``` +yarn install +``` + +Prepare your terminal session with AWS credentials for use with AWS CDK -## Deploy Blue Green +``` +# Personally I use aws-vault, configure your terminal using your workflow +aws-vault exec my-cool-aws-profile +``` -Deploy AWS stack +[Find your public IP](https://www.google.com/search?q=whats+my+ip) and update `externalAccess` in [`ip.ts`](./lib/ip.ts): ``` -yarn cdk --app "yarn ts-node bin/bluegreen.ts" deploy +export const externalAccess = Peer.ipv4("1.2.3.4/32"); ``` -Retrieve the load balancer hostname from the CloudFormation Outputs of the stack and use it to run the client +## Express.js App + +This repo includes a simple Express.js [application](./expressjs/lib/app.ts) that is run on ECS Fargate to demonstrate ECS Service Connect and App Mesh. + +It has the following endpoints: +- `GET /`: status endpoint that responds with JSON specifying the `service` and `version` that have been configured by environment + ``` + GET / + + {"message":"ok","service":"green","version":"1"} + ``` +- `GET /downstream/:service`: fetches a downstream request to `service` and responds with the result + ``` + GET /downstream/other-service + + {"message":"fetched!","service":"green","response":{"message":"ok","service":"other-service","version":"unknown"}} + ``` +You can run the application locally to understand it better: ``` -> yarn start blueg-MeshG-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com -❯ yarn start blueg-MeshG-199JCBZL6CQMI-54586324aeeb125f.elb.ap-southeast-2.amazonaws.com +❯ cd expressjs +❯ yarn install +❯ SERVICE_NAME=testing SERVICE_VERSION=10 yarn start yarn run v1.22.15 -$ ts-node bin/client.ts blueg-MeshG-199JCBZL6CQMI-54586324aeeb125f.elb.ap-southeast-2.amazonaws.com -Running against http://blueg-MeshG-199JCBZL6CQMI-54586324aeeb125f.elb.ap-southeast-2.amazonaws.com/service-blue-green -200 OK Green:3=1 -200 OK Green:3=2 -200 OK Blue:4=1 Green:3=2 -200 OK Blue:4=1 Green:3=3 -200 OK Blue:4=2 Green:3=3 -200 OK Blue:4=3 Green:3=3 -200 OK Blue:4=3 Green:3=4 -200 OK Blue:4=3 Green:3=5 -200 OK Blue:4=4 Green:3=5 +$ ts-node bin/serve.ts +Service 'testing' (version 10) running on port: 3000 +``` + +## ECS Service Connect + +[ECS Service Connect](https://aws.amazon.com/blogs/aws/new-amazon-ecs-service-connect-enabling-easy-communication-between-microservices/) provides an out-of-the-box service discovery method for services running in ECS. + +In essence, for developer in connect their service to a "namespace" they just need to: +- configure the ECS Service Connect namespace when they set up an ECS Cluster +- provide some additional discovery config to their ECS Task Definitions + +Under the hood, my investigation leads me to believe that ECS Service Connect configures and manages App Mesh for you. With App Mesh the Envoy _control plane_ is managed and the developer is expected to manage the Envoy _data plane_ by configuring proxy sidecars in their services, but ECS Service Connect uses the new discovery config to configure and inject the Envoy data plane _as well_. Below is a sample startup log line from the proxy container in an ECS Service Connect-configured service that indicates that the provided proxy container is connecting to an AWS-managed App Mesh: + +``` +time="2022-12-10T11:24:48Z" level=info msg="App Mesh Environment Variables: [APPMESH_XDS_ENDPOINT=unix:///var/run/ecs/appnet/relay/appnet_relay_listener.sock APPMESH_METRIC_EXTENSION_VERSION=1 APPMESH_RESOURCE_ARN=arn:aws:ecs:ap-southeast-2:933397847440:task-set/tomwrightmesh-cluster/tomwrightmesh-ServiceAService01F8F99F-QP8c9w6ArTf1/ecs-svc/2740018079984326164]" +``` + +### Exploring ECS Service Connect with AWS CDK + +Check out [`serviceconnect.ts`](./lib/serviceconnect.ts) to see the implementation of the ECS Service Connect architecture. The configuration of components is minimal, primarily just "wiring things up". + +![Service Connect Infrastructure Diagram](./docs/serviceconnect.svg) + +Deploy our `serviceconnect` example: + +``` +yarn cdk:serviceconnect deploy +``` + +Open up the AWS Console > Navigate to ECS > `serviceconnect` ECS Cluster > Find `green` service > Find running task > Get Public IP of task + +Connect to the service in your browser: + +``` +http:/// + +{"message":"ok","service":"green","version":"unknown"} +``` + +Our services are able to discovery and intercommunicate through the Envoy proxy that has been configured and added by ECS Service Connect. Connect to “downstream” `blue` service from `green` service via ECS Service Connect (service discovery name `blue` in namespace `serviceconnect` = `blue.serviceconnect`) + +``` +http:///downstream/blue.serviceconnect + +{"message":"fetched!","service":"green","response":{"message":"ok","service":"blue","version":"unknown"}} +``` + +Clean up the resources used for our `serviceconnect` example: + +``` +yarn cdk:serviceconnect destroy +``` + +The Cloud Map namespace created by ECS Service Connect is **not automatically deleted** so clean that up manually. AWS Console > Cloud Map service > `serviceconnect` namespace > Delete + +### What did we learn? + +ECS Service Connect allows us to enable service discovery between our ECS Services. To do so, we only need to add configuration to our ECS Clusters, Task Definitions and Services. + +This service discovery by ECS Service Connect: +- could replace internal load balancers between our services, eliminating those costs. +- produces additional networking telemetry, providing us more observability in our traffic + +However, the App Mesh managed internally by ECS Service Connect is not accessible by us for additional configuration. This prevents us from leveraging the more advanced features of an Envoy service mesh: +- complex routing and load balancing based on paths or headers + +The limited feature set and lack of extensibility of ECS Service Connect makes it less suitable for complex architectures. + +## App Mesh + +[AWS App Mesh](https://docs.aws.amazon.com/app-mesh/latest/userguide/what-is-app-mesh.html) provides a managed service mesh control plane based on [Envoy](https://www.envoyproxy.io/) proxy. The service mesh has integrations with Cloud Map and Route 53 for service discovery. The App Mesh components of the service mesh can be managed by AWS APIs, CloudFormation, and AWS CDK. + +Developers are expected to add and configure a proxy sidecar container (the Envoy _data plane_) to their ECS Services. This proxy connects to the App Mesh _control plane_ and receives routing information about the logical components of the mesh. The proxy then uses service discovery via Cloud Map or Route 53 to populate the real network destinations and send traffic. + +![Components in App Mesh](./docs/appmesh-components.svg) + +Understanding how the App Mesh components fit together can be confusing. Hopefully this diagram and table helps describe how the logical components of an App Mesh mesh relate to actual running services on the network. + +| Component | Description | Connects to | Is backed by running service | +| --------------- | -------------------------------------- | ------------------------ | ---------------------------- | +| Virtual Gateway | Receives traffic from outside the mesh | Virtual Routes | Yes | +| Virtual Route | Directs traffic to services | Virtual Services | No | +| Virtual Service | Destination of traffic in a mesh | Virtual Nodes or Routers | No | +| Virtual Router | Directs traffic to route | Virtual Routes | No | +| Virtual Node | Responds to and consumes traffic | Virtual Services | Yes | + + +### Exploring App Mesh with AWS CDK + +Configuring App Mesh ourselves is significantly more complex than with ECS Service Connect. Check out [`appmesh.ts`](./lib/appmesh.ts) to see the implementation of the below App Mesh architecture. + +#### HTTP-Only vs Private DNS Cloud Map namespace + +Despite the fact that the Envoy proxy will handle routing traffic as per service discovery, the running service (that knows nothing about Envoy being present) still needs to be able to resolve each DNS name (e.g. `blue.appmesh`) and attempt to send it _somewhere_ for it to be intercepted by the Envoy proxy. This situation is described in [this troubleshooting guide](https://docs.aws.amazon.com/app-mesh/latest/userguide/troubleshooting-connectivity.html). + +In this example implementation the easiest way to make sure this works (and also to provide better interoperability with resources outside the service mesh) is to configure the Cloud Map namespace as a Private DNS one so that Route 53 records are published when services are registered. + +However, ECS Service Connect doesn't do this, and uses only an HTTP-Only Cloud Map namespace (service addresses are only stored in Cloud Map and not published to Route 53). This should mean that running service in ECS is unable to resolve the DNS name of the service but that doesn't seem to be the case. + +_[?] Haven't gotten to the bottom of this one yet [?]_ + +![App Mesh Infrastructure Diagram](./docs/appmesh.svg) + +Deploy our `appmesh` example: + +``` +yarn cdk:appmesh deploy +``` + +Open up the AWS Console > Navigate to ECS > `appmesh` ECS Cluster > Find `green` service > Find running task > Get Public IP of task + +Connect to the service in your browser: + +``` +http:/// + +{"message":"ok","service":"green","version":"unknown"} +``` + +Our services are able to discovery and intercommunicate through the Envoy proxy that has been configured and added by App Mesh. Connect to “downstream” `blue` service from `green` service via App Mesh (service discovery name `blue` in namespace `appmesh` = `blue.appmesh`) + +``` +http:///downstream/blue.appmesh + +{"message":"fetched!","service":"green","response":{"message":"ok","service":"blue","version":"unknown"}} +``` + +This is all the same as before. Now for something a little different: In the CloudFormation Outputs of the `appmesh` stack, locate the hostname of our gateway load balancer and explore connecting to services via the "front door": + +``` +http://appme-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com/blue + +{"message":"ok","service":"blue","version":"unknown"} + +http://appme-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com/green + +{"message":"ok","service":"green","version":"unknown"} +``` + +Now try using our `/split` route that is internally load balanced between our `blue` and `green` services by the service mesh. Keep in mind there is no load balancer involved in this routing, the Envoy gateway is routing based on the configuration provided by the App Mesh control plane. + +``` +http://appme-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com/split + +{"message":"ok","service":"blue","version":"unknown"} +{"message":"ok","service":"green","version":"unknown"} +{"message":"ok","service":"green","version":"unknown"} +{"message":"ok","service":"blue","version":"unknown"} +``` + +We can still traverse the service mesh to `blue` using our `/downstream` route on our Express.js application: + +``` + +http://appme-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com/split/downstream/blue.app + +gateway > blue > blue +gateway > green > blue + +{"message":"fetched!","service":"green","response":{"message":"ok","service":"blue","version":"unknown"}} +{"message":"fetched!","service":"blue","response":{"message":"ok","service":"blue","version":"unknown"}} +{"message":"fetched!","service":"blue","response":{"message":"ok","service":"blue","version":"unknown"}} +``` +### What did we learn? + +Configuring App Mesh requires complex boilerplate. AWS CDK can help us abstract away through Constructs. + +Having control over the App Mesh service mesh allows us to configure flexible routing behaviour within the service mesh without the need for load balancers. + +## Blue-Green deployments using stateful AWS CDK + +This one is just a bonus -- an idea I stumbled on while writing the CDK code for this investigation. + +Blue-Green deployments with AWS CDK are difficult to implement because blue-green inherently has state involved -- which stack is current and which one should I update next? + +This proof-of-concept solves this using state maintained in AWS Parameter Store. The AWS CDK retrieves this state and applies logic on it to perform the synth, and then the deployment handles updating this state as part of the deployment. + +Explore the code for the blue-green architecture below starting from [`bluegreen.ts`](./bin/bluegreen.ts). The ECS and App Mesh portions of this implementation are derived from the `appmesh` example above so refer there if anything is unclear. + +![Blue-Green Infrastructure Diagram](./docs/bluegreen.svg) + +Deploy our example `bluegreen` infrastructure: + +``` +yarn cdk:bluegreen deploy +``` + +Retrieve the load balancer hostname from output of CloudFormation stack and use that to connect, refresh a few times to see load balancing. Both `blue` and `green` have had an initial deployment of `version: 1`: + +``` +http://appme-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com + +{"message":"ok","service":"blue","version":"1"} +{"message":"ok","service":"green","version":"1"} +``` + +Run the included client script to collect request counts against the blue-green setup using `yarn start ` + +``` +❯ yarn start blueg-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com +yarn run v1.22.15 +$ ts-node bin/client.ts blueg-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com +Running against http://blueg-meshg-xxxx-xxxx.elb.ap-southeast-2.amazonaws.com +200 OK green:1=1 +200 OK blue:1=1 green:1=1 +200 OK blue:1=1 green:1=2 +200 OK blue:1=1 green:1=3 ... ``` + +Update [`bluegreen.ts`](./lib/bluegreen.ts) to bump the version to `2`. Inspect the `cdk diff` to see that only `green` is bumping to `version: 2` and the Parameter Store state is being updated to reflect that: + +``` +❯ yarn cdk:bluegreen diff +yarn run v1.22.15 +$ cdk --app "yarn ts-node bin/bluegreen.ts" diff +$ /Users/tomwright/Projects/_cultureamp/meshtest/node_modules/.bin/ts-node bin/bluegreen.ts +Stack bluegreen/bluegreen +Resources +[~] AWS::SSM::Parameter bluegreen/Deploy/StateParameter DeployStateParameter8F1DC08B + └─ [~] Value + ├─ [-] {"nextUpdate":"blue","currentVersion":1,"nextVersion":1} + └─ [+] {"nextUpdate":"green","nextVersion":2,"currentVersion":1} +[~] AWS::ECS::TaskDefinition bluegreen/Deploy/Green/Service/TaskDefinition DeployGreenServiceTaskDefinition773C6E41 replace + └─ [~] ContainerDefinitions (requires replacement) + └─ @@ -20,7 +20,7 @@ + [ ] }, + [ ] { + [ ] "Name": "SERVICE_VERSION", + [-] "Value": "1" + [+] "Value": "2" + [ ] } + [ ] ], + [ ] "Essential": true, +``` + +Run the deployment: + +``` +yarn cdk:bluegreen deploy +``` + +Watch the stats from the client script and wait for requests from `green:2` to appear: + +``` +200 OK blue:1=74 green:1=76 +200 OK blue:1=75 green:1=76 +200 OK blue:1=76 green:1=76 +200 OK blue:1=77 green:1=76 +200 OK blue:1=78 green:1=76 +200 OK blue:1=78 green:1=76 green:2=1 <-- new version appears running in green +200 OK blue:1=78 green:1=76 green:2=2 +200 OK blue:1=79 green:1=76 green:2=2 +200 OK blue:1=79 green:1=77 green:2=2 +200 OK blue:1=80 green:1=77 green:2=2 +200 OK blue:1=81 green:1=77 green:2=2 <--- green:1 no longer receiving requests +``` + +Update again to `version: 3` and `cdk diff` to see that now now `blue` has been updated for deployment: + +``` +❯ yarn cdk:bluegreen diff +yarn run v1.22.15 +$ cdk --app "yarn ts-node bin/bluegreen.ts" diff +$ /Users/tomwright/Projects/_cultureamp/meshtest/node_modules/.bin/ts-node bin/bluegreen.ts +Stack bluegreen/bluegreen +Resources +[~] AWS::SSM::Parameter bluegreen/Deploy/StateParameter DeployStateParameter8F1DC08B + └─ [~] Value + ├─ [-] {"nextUpdate":"green","nextVersion":2,"currentVersion":1} + └─ [+] {"nextUpdate":"blue","nextVersion":3,"currentVersion":2} +[~] AWS::ECS::TaskDefinition bluegreen/Deploy/Blue/Service/TaskDefinition DeployBlueServiceTaskDefinition7C878561 replace + └─ [~] ContainerDefinitions (requires replacement) + └─ @@ -20,7 +20,7 @@ + [ ] }, + [ ] { + [ ] "Name": "SERVICE_VERSION", + [-] "Value": "1" + [+] "Value": "3" + [ ] } + [ ] ], + [ ] "Essential": true, +``` + +Deploy and watch client stats and `blue:3` will start serving: + +... +200 OK blue:1=56 green:2=38 +200 OK blue:1=56 green:2=39 +200 OK blue:1=56 green:2=40 +200 OK blue:1=56 blue:3=1 green:2=40 <--- blue starts serving version 3 +200 OK blue:1=57 blue:3=1 green:2=40 +200 OK blue:1=57 blue:3=2 green:2=40 +... +200 OK blue:1=69 blue:3=14 green:2=54 +200 OK blue:1=70 blue:3=14 green:2=54 +200 OK blue:1=70 blue:3=15 green:2=54 +200 OK blue:1=70 blue:3=15 green:2=55 +200 OK blue:1=70 blue:3=16 green:2=55. <--- blue:1 stops receiving requests +... +``` + +Clean up `bluegreen` example + +``` +yarn cdk:bluegreen destroy +``` \ No newline at end of file diff --git a/bin/appmesh.ts b/bin/appmesh.ts index 7142e25..f98b36c 100644 --- a/bin/appmesh.ts +++ b/bin/appmesh.ts @@ -1,9 +1,9 @@ #!/usr/bin/env node import "source-map-support/register"; -import { Peer } from "aws-cdk-lib/aws-ec2"; import { AppMeshApp } from "../lib/appmesh"; +import { externalAccess } from "../lib/ip"; new AppMeshApp({ namespaceName: "appmesh", - externalAccess: Peer.ipv4("139.130.21.126/32"), + externalAccess, }); diff --git a/bin/bluegreen.ts b/bin/bluegreen.ts index c5ff29b..f086acd 100644 --- a/bin/bluegreen.ts +++ b/bin/bluegreen.ts @@ -1,17 +1,27 @@ #!/usr/bin/env node import "source-map-support/register"; -import { Peer } from "aws-cdk-lib/aws-ec2"; import { BlueGreenApp } from "../lib/bluegreen"; import { App } from "aws-cdk-lib"; import { injectBlueGreenState as injectBlueGreenState } from "../lib/bluegreen/deployment"; +import { externalAccess } from "../lib/ip"; async function main() { + /** + * as constructors cannot be async and retrieving the current + * state from AWS Parameter Store is async it has to be done + * here + */ + const app = new App(); await injectBlueGreenState(app); // needs to be performed out here because async unsupported in constructors + /** + * having populated the state we can now enter the construct tree + */ + new BlueGreenApp(app, "bluegreen", { namespaceName: "bluegreen", - externalAccess: Peer.ipv4("139.130.21.126/32"), + externalAccess, }); } diff --git a/bin/serviceconnect.ts b/bin/serviceconnect.ts index c675b2f..b6c598f 100644 --- a/bin/serviceconnect.ts +++ b/bin/serviceconnect.ts @@ -2,8 +2,9 @@ import "source-map-support/register"; import { Peer } from "aws-cdk-lib/aws-ec2"; import { ServiceConnectApp } from "../lib/serviceconnect"; +import { externalAccess } from "../lib/ip"; new ServiceConnectApp({ namespaceName: "serviceconnect", - externalAccess: Peer.ipv4("139.130.21.126/32"), + externalAccess, }); diff --git a/docs/appmesh-components.svg b/docs/appmesh-components.svg new file mode 100644 index 0000000..d94ab51 --- /dev/null +++ b/docs/appmesh-components.svg @@ -0,0 +1,3 @@ + + +
Node
Node
Router
Router
Service
Service
Mesh
Mesh
Running
service
Running...
Running
service
Running...
Gateway
Gateway
Route
Route
Node
Node
Running
service
Running...
Route
Route
Service
Service
Service
Service
Components of App Mesh
Components of App M...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/appmesh.svg b/docs/appmesh.svg new file mode 100644 index 0000000..2059d80 --- /dev/null +++ b/docs/appmesh.svg @@ -0,0 +1,3 @@ + + +
VPC
VPC
Client
Client
ECS Fargate
ECS Fargate
green
green
Envoy Proxy
green.appmesh
Envoy Pr...
Envoy Proxy
blue.appmesh
Envoy Pr...
ECS Fargate
ECS Fargate
blue
blue
App Mesh
App Mesh
App Mesh
App Mesh
blue = 10.0.0.25
green = 10.0.0.24
gateway = 10.0.0.23
blue = 10.0.0.25...
Route53 Private
Hosted Zone
Route53 Priva...
Envoy (ECS Fargate)
Envoy (ECS Fa...
gateway
gateway
Gateway
gateway
Gateway...
blue
blue
green
green
CloudMap Namespace
CloudMap Name...
gateway
gateway
Network
Load Balancer
Network...
Node
blue.appmesh
Node...
Node
green.appmesh
Node...
/blue
/blue
/green
/green
Service
router.appmesh
Service...
Router
Router
Service
green.appmesh
Service...
Service
blue.appmesh
Service...
/split
/splitViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/bluegreen.svg b/docs/bluegreen.svg new file mode 100644 index 0000000..b37318b --- /dev/null +++ b/docs/bluegreen.svg @@ -0,0 +1,3 @@ + + +
VPC
VPC
Client
Client
ECS Fargate
ECS Fargate
green
green
Envoy Proxy
green.appmesh
Envoy Pr...
Envoy Proxy
blue.appmesh
Envoy Pr...
ECS Fargate
ECS Fargate
blue
blue
Blue-Green
Blue-Green
App Mesh
App Mesh
Envoy (ECS Fargate)
Envoy (ECS Fa...
gateway
gateway
Network
Load Balancer
Network...
see App Mesh
see App Mesh
CDK
CDK
CloudFormation
CloudFormation
Parameter Store
Parameter Sto...
next = green
current version = 3
prev version = 2
next = green...
next = blue
current version = 2
prev version = 1
next = blue...
version = 2
version = 2
version = 3
version = 3Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/serviceconnect.svg b/docs/serviceconnect.svg new file mode 100644 index 0000000..289c406 --- /dev/null +++ b/docs/serviceconnect.svg @@ -0,0 +1,3 @@ + + +
ECS Service Connect
ECS Service Connect
VPC
VPC
Client
Client
ECS Fargate
ECS Fargate
green
green
ECS Fargate
ECS Fargate
blue
blue
ECS Service Connect
ECS Service Connect
blue = 10.0.0.25
green = 10.0.0.24
blue = 10.0.0.25...
CloudMap Namespace
CloudMap Name...
ECS Service Connect
(Managed App Mesh)
ECS Service C...
Envoy Proxy
Envoy Pr...
Envoy Proxy
Envoy Pr...
blue.serviceconnect
green.serviceconnect
blue.serviceconnect...
blue
blue
green
greenViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/lib/appmesh.ts b/lib/appmesh.ts index 3c1ba8f..3d9e49b 100644 --- a/lib/appmesh.ts +++ b/lib/appmesh.ts @@ -25,8 +25,20 @@ export class AppMeshApp extends App { const { namespaceName, externalAccess } = props; + // create ourselves an empty stack to hold our resources + const stack = new Stack(this, namespaceName); + /** + * create our ECS Cluster that is configured for App Mesh + * + * the key difference is that the namespace is backed by Route 53 + * + * `cluster` our ECS Cluster with underlying default VPC + * `namespace` our Cloud Map namespace created by ourselves + * `securityGroup` security group accessible from our IP to make things easy + **/ + const { cluster, namespace, securityGroup } = new AppMeshCluster( stack, "Cluster", @@ -36,6 +48,15 @@ export class AppMeshApp extends App { } ); + /** + * configure App Mesh components + * + * `mesh` the App Mesh mesh that services will be added to as nodes, + * routes, services, etc. + * `gateway` the App Mesh Gateway, running on ECS Fargate, that will + * receive external traffic and route it into the service mesh + */ + const { mesh, gateway } = new AppMesh(stack, "Mesh", { cluster, securityGroup, @@ -49,6 +70,21 @@ export class AppMeshApp extends App { securityGroup, }; + /** + * create two ECS Services, "blue" and "green", running our sample Express.js + * application that are configured for App Mesh + * + * the details of configuring for App Mesh are quite gory -- take a look + * inside the Construct! + * + * thankfull this is precisely the sort of thing that AWS CDK excels at, + * abstracting away gory details + * + * we can use the /downstream endpoint of our sample application to explore + * how these services are able to discover and route traffic to each other + * as they have been connected to the same namespace + **/ + const blue = new AppMeshExpress(stack, "Blue", { serviceName: "blue", ...meshThings, @@ -59,10 +95,22 @@ export class AppMeshApp extends App { ...meshThings, }); + /** + * configure each service as a "backend" of the other's node so that + * traffic can be routed between them via the mesh + */ + blue.virtualNode.addBackend(Backend.virtualService(green.virtualService)); green.virtualNode.addBackend(Backend.virtualService(blue.virtualService)); - // configure gateway routes for the services + /** + * configure a route on the gateway for each service so that + * traffic entering the mesh is routed to that service + * + * this sets up path-based routing: + * - all traffic with path starting with '/blue' -> blue + * - all traffic with path starting with '/green' -> green + */ gateway.addGatewayRoute("blue", { routeSpec: GatewayRouteSpec.http({ @@ -82,7 +130,10 @@ export class AppMeshApp extends App { }), }); - // configure fancy routing based on path + /** + * configure a router that divides traffic equally between our + * "blue" and "green" nodes + */ const router = new VirtualRouter(stack, "Router", { mesh, @@ -106,6 +157,12 @@ export class AppMeshApp extends App { }), }); + /** + * wrap our router in a service so that it can be the destination of + * a gateway route: + * - all traffic with path starting with '/split' -> router + */ + const routerService = new VirtualService(stack, "RouterService", { virtualServiceProvider: VirtualServiceProvider.virtualRouter(router), virtualServiceName: "router", diff --git a/lib/appmesh/cluster.ts b/lib/appmesh/cluster.ts index 8bfacbe..f6d6940 100644 --- a/lib/appmesh/cluster.ts +++ b/lib/appmesh/cluster.ts @@ -26,10 +26,27 @@ export class AppMeshCluster extends Construct { const { namespaceName } = props; + /** + * Vanilla ECS Cluster with underlying default VPC + */ + this.cluster = new Cluster(this, `Cluster`, { clusterName: namespaceName, }); + /** + * create our own Cloud Map namespace for service discovery + * + * notably, this is a Private DNS namespace so Cloud Map creates + * an associated Route 53 private hosted zone and automates + * managing the records in it + * + * this differs from ECS Service Connect in that it uses an HTTP-Only + * namespace with no backing Route 53 DNS records + * + * see additional notes in the README + */ + this.namespace = new PrivateDnsNamespace(this, "Namespace", { name: namespaceName, vpc: this.cluster.vpc, diff --git a/lib/appmesh/express.ts b/lib/appmesh/express.ts index 99f7eba..a330892 100644 --- a/lib/appmesh/express.ts +++ b/lib/appmesh/express.ts @@ -43,6 +43,16 @@ export class AppMeshExpress extends Construct { const { cluster, mesh } = props; + /** + * our task definition needs to have proxy configuration applied + * + * this is almost entirely boilerplate except for 'appPort' which + * indicates which port our running service will be on + * + * (app mesh needs to know this port so that it can 'hijack' it + * for the proxying) + */ + const taskDefinition = new FargateTaskDefinition(this, "TaskDefinition", { proxyConfiguration: new AppMeshProxyConfiguration({ containerName: "proxy", @@ -56,6 +66,14 @@ export class AppMeshExpress extends Construct { }), }); + /** + * configure and add our application -- note there is nothing + * "service meshy" about it! + * + * unbeknownst to our little application, all sort of whacky + * stuff is happening elsewhere... + */ + const expressContainer = taskDefinition.addContainer("expressjs", { image: ContainerImage.fromAsset("expressjs"), command: ["serve:js"], @@ -86,6 +104,10 @@ export class AppMeshExpress extends Construct { assignPublicIp: true, }); + /** + * register our service for service discovery with Cloud Map + */ + const cloudMapService = new Service(this, "ServiceDiscovery", { name: props.serviceName, namespace: props.namespace, @@ -99,6 +121,16 @@ export class AppMeshExpress extends Construct { service: cloudMapService, }); + /** + * create a logical node and service in our service mesh for our + * Express.js application + * + * note how service discovery is "wired up" on the virtual node + * to our Cloud Map service that we just registered + * + * ECS -> Cloud Map -> App Mesh + */ + const virtualNode = new VirtualNode(this, "VirtualNode", { mesh, serviceDiscovery: ServiceDiscovery.cloudMap(cloudMapService), @@ -113,6 +145,22 @@ export class AppMeshExpress extends Construct { virtualNode.grantStreamAggregatedResources(taskDefinition.taskRole); this.virtualNode = virtualNode; + const virtualService = new VirtualService(this, "VirtualService", { + virtualServiceName: `${props.serviceName}.${props.namespace.namespaceName}`, + virtualServiceProvider: VirtualServiceProvider.virtualNode(virtualNode), + }); + this.virtualService = virtualService; + + /** + * where the magic happens -- configure an Envoy proxy sidecar in our task + * definition + * + * note again the use of APPMESH_RESOURCE_ARN to configure this proxy + * as the virtual node we added + * + * the rest is _entirely_ boilerplate :sweat: + */ + const proxyContainer = taskDefinition.addContainer("proxy", { image: ContainerImage.fromRegistry( "public.ecr.aws/appmesh/aws-appmesh-envoy:v1.24.0.0-prod" @@ -156,11 +204,5 @@ export class AppMeshExpress extends Construct { expressContainer.addContainerDependencies({ container: proxyContainer, }); - - const virtualService = new VirtualService(this, "VirtualService", { - virtualServiceName: `${props.serviceName}.${props.namespace.namespaceName}`, - virtualServiceProvider: VirtualServiceProvider.virtualNode(virtualNode), - }); - this.virtualService = virtualService; } } diff --git a/lib/appmesh/mesh.ts b/lib/appmesh/mesh.ts index bb85ed4..66f3bad 100644 --- a/lib/appmesh/mesh.ts +++ b/lib/appmesh/mesh.ts @@ -33,11 +33,22 @@ export class AppMesh extends Construct { const { cluster, securityGroup } = props; + /** + * create the App Mesh mesh + */ + const mesh = new Mesh(this, "Mesh", { egressFilter: MeshFilterType.DROP_ALL, }); this.mesh = mesh; + /** + * configure the gateway within the mesh + * + * at this stage this is just a logical component of the mesh + * with no corresponding running infrastructure + */ + const gateway = new VirtualGateway(this, "VirtualGateway", { mesh, listeners: [ @@ -53,6 +64,21 @@ export class AppMesh extends Construct { }); this.gateway = gateway; + /** + * configure a task definition that runs only the + * Envoy proxy + * + * we "wire this up" to the gateway in the mesh using the + * APPMESH_RESOURCE_ARN environment variable + * + * the rest is essentially boilerplate -- wordy! + * + * note: for nodes in the mesh, it would instead be our application + * with the Envoy proxy as a sidecar -- but a gateway is + * just the Envoy itself expecting to receive traffic directly + * and forward it + */ + const gatewayTaskDefinition = new FargateTaskDefinition( this, "GatewayTaskDefinition" @@ -91,6 +117,11 @@ export class AppMesh extends Construct { }), }); + /** + * ensure that the Envoy when it starts has permissions + * to connect to AppMesh and download the state of the mesh + */ + gatewayTaskDefinition.addToTaskRolePolicy( new PolicyStatement({ effect: Effect.ALLOW, @@ -99,6 +130,13 @@ export class AppMesh extends Construct { }) ); + /** + * alright some actual infrastructure! + * + * fire up a Fargate service behind an NLB to act as the "front door" + * of our service mesh -- external traffic comes in here + */ + const gatewayService = new NetworkLoadBalancedFargateService( this, "GatewayService", diff --git a/lib/bluegreen.ts b/lib/bluegreen.ts index 0235c53..2a4f735 100644 --- a/lib/bluegreen.ts +++ b/lib/bluegreen.ts @@ -26,8 +26,14 @@ export class BlueGreenApp extends Construct { const { namespaceName, externalAccess } = props; + // create a stack to hold all our resources + const stack = new Stack(this, namespaceName); + /** + * set up for a App Mesh as per last time + */ + const { cluster, namespace, securityGroup } = new AppMeshCluster( stack, "Cluster", @@ -50,6 +56,21 @@ export class BlueGreenApp extends Construct { securityGroup, }; + /** + * construct a BlueGreenDeployment, passing the current version + * + * if this is already deployed this version will be partnered with + * the previous version stored in Parameter Store state + * + * the `build` prop is passed a function used to construct both the + * blue and green sides of the deployment, see BlueGreenDeployment + * for details + * + * for this example, our blue-green deployment is simply one instance + * of our Express.js app wired into App Mesh + * + */ + const deployment = new BlueGreenDeployment(stack, "Deploy", { version: 1, build: (scope, version) => { @@ -62,7 +83,10 @@ export class BlueGreenApp extends Construct { }, }); - // construct router on top of services + /** + * construct a router on top of our two nodes exposed by the + * BlueGreenDeployment and connect that to our gateway + */ const router = new VirtualRouter(stack, "Router", { mesh, diff --git a/lib/bluegreen/deployment.ts b/lib/bluegreen/deployment.ts index 30c2730..f3f3744 100644 --- a/lib/bluegreen/deployment.ts +++ b/lib/bluegreen/deployment.ts @@ -7,13 +7,35 @@ import { VirtualNode } from "aws-cdk-lib/aws-appmesh"; export const BLUE_GREEN_STATE = "blue-green-state"; +/** + * retrieves the state from Parameter Store and uses the + * `setContext` API to inject the state into the App + * + * if the context already exists then this procedure is skipped, + * this allows for the context to be injected via some other means + * through `cdk.context.json` + * + * currently this proof-of-concept just uses a fixed parameter + * name and would need to be extended with some sort of naming + * schema + * + * @param app to inject context into with node.setContext API + */ export async function injectBlueGreenState(app: App) { let state: BlueGreenState | null = null; + /** + * check for state already being populated + */ + const existingContext = app.node.tryGetContext(BLUE_GREEN_STATE); if (existingContext) { state = existingContext; } else { + /** + * perform retrieval of state from Parameter Store + */ + const ssm = new SSM({}); try { @@ -29,6 +51,11 @@ export async function injectBlueGreenState(app: App) { } catch (e) {} } + /** + * if at this state we still have no state, assume + * some defaults to allow for the stack to be bootstrapped + */ + if (!state) { state = { nextUpdate: "blue", @@ -37,6 +64,10 @@ export async function injectBlueGreenState(app: App) { }; } + /** + * inject state into root node using the Context API + */ + app.node.setContext(BLUE_GREEN_STATE, state); } @@ -57,19 +88,38 @@ export class BlueGreenDeployment extends Construct { constructor(scope: Construct, id: string, props: Props) { super(scope, id); - // manage state + /** + * expect to retrieve our state from the Context API + * + * you mind suspect that this is the perfect use case for something + * like `ssm.StringParameter.fromLookup()` but it doesn't work as expected! + * + * context providers from lookup actually execute _after_ construction + * of the construct tree and so you are unable to leverage that data for + * code logic! + * + * hence we rely on the Context API as populated in ./bin/bluegreen.ts + * prior to entering the construct tree + * + * see: https://github.com/aws/aws-cdk/issues/8273#issuecomment-824801527 + */ - // using ss.StringParameter.fromLookup() doesn't work as expected! - // Context providers actually execute _after_ construction - // So leveraging data plugged into Context directly instead - // see: https://github.com/aws/aws-cdk/issues/8273#issuecomment-824801527 const contextData = scope.node.tryGetContext(BLUE_GREEN_STATE); if (!contextData) { throw new Error("Unable to operate without context"); } const currentState: BlueGreenState = contextData; - // if the version has updated, flip stacks and update versions + /** + * perform our core logic on state and the provided version and + * construct the resource to manage it in Parameter Store + * + * if the version has changed, then flip our update stack and + * rotate versions + * + * else retain the current state, producing an empty diff + */ + let newState = currentState; if (currentState.currentVersion != props.version) { newState = { @@ -84,10 +134,10 @@ export class BlueGreenDeployment extends Construct { stringValue: JSON.stringify(newState), }); - // construct services - - const blue = new Construct(this, "Blue"); - const green = new Construct(this, "Green"); + /** + * now determine the appropriate versions of blue and green + * based on our state + */ const blueVersion = newState.nextUpdate == "blue" @@ -98,7 +148,24 @@ export class BlueGreenDeployment extends Construct { ? newState.currentVersion : newState.previousVersion; + /** + * produce a construct for blue and green + * and run our provided .build function + * against each to build our app + * + * the appropriate version is passed -- this doesn't + * have to be a value and could really be any data that + * we want to track in state + */ + + const blue = new Construct(this, "Blue"); + const green = new Construct(this, "Green"); + this.blue = props.build(blue, blueVersion); this.green = props.build(green, greenVersion); + + /** + * + */ } } diff --git a/lib/ip.ts b/lib/ip.ts new file mode 100644 index 0000000..964e6f5 --- /dev/null +++ b/lib/ip.ts @@ -0,0 +1,3 @@ +import { Peer } from "aws-cdk-lib/aws-ec2"; + +export const externalAccess = Peer.ipv4("139.218.174.36/32"); diff --git a/lib/serviceconnect.ts b/lib/serviceconnect.ts index a4b47a6..fd258a6 100644 --- a/lib/serviceconnect.ts +++ b/lib/serviceconnect.ts @@ -14,8 +14,18 @@ export class ServiceConnectApp extends App { const { namespaceName, externalAccess } = props; + // create ourselves an empty stack to hold our resources + const stack = new Stack(this, namespaceName); + /** + * create our ECS Cluster that is configured to use ECS Service Connect + * + * `cluster` our ECS Cluster with underlying default VPC + * `namespace` our Cloud Map namespace created by ECS Service Connect + * `securityGroup` security group accessible from our IP to make things easy + **/ + const { cluster, namespace, securityGroup } = new ServiceConnectCluster( stack, "Cluster", @@ -25,6 +35,15 @@ export class ServiceConnectApp extends App { } ); + /** + * create two ECS Services, "blue" and "green", running our sample Express.js + * application that are configured to use ECS Service Connect + * + * we can use the /downstream endpoint of our sample application to explore + * how these services are able to discover and route traffic to each other + * as they have been connected to the same namespace + **/ + new ServiceConnectExpress(stack, "Blue", { serviceName: "blue", cluster, diff --git a/lib/serviceconnect/cluster.ts b/lib/serviceconnect/cluster.ts index e8ccd10..1309fc9 100644 --- a/lib/serviceconnect/cluster.ts +++ b/lib/serviceconnect/cluster.ts @@ -18,6 +18,12 @@ export class ServiceConnectCluster extends Construct { const { namespaceName } = props; + /** + * defaultCloudMapNamespace allows us to configure the ECS Cluster + * for use with ECS Service Connect + * + * ECS creates a HTTP-only Cloud Map namespace automatically + */ this.cluster = new Cluster(this, `Cluster`, { clusterName: namespaceName, defaultCloudMapNamespace: { @@ -26,6 +32,7 @@ export class ServiceConnectCluster extends Construct { }, }); + // ensure that default namespace has been set and expose on the construct if (this.cluster.defaultCloudMapNamespace) { this.namespace = this.cluster.defaultCloudMapNamespace; } else { diff --git a/lib/serviceconnect/express.ts b/lib/serviceconnect/express.ts index bfd5e14..ad03cc5 100644 --- a/lib/serviceconnect/express.ts +++ b/lib/serviceconnect/express.ts @@ -24,6 +24,14 @@ export class ServiceConnectExpress extends Construct { const { cluster, namespace, serviceName, securityGroup } = props; + /** + * preparing a task definition for use with ECS Service Connect requires + * adding some details to our `portMappings` + * + * see `name` and `appProtocol` added below that allow ECS Service Connect make + * that container port discoverable + */ + const taskDefinition = new FargateTaskDefinition(this, "TaskDefinition"); taskDefinition.addContainer("expressjs", { image: ContainerImage.fromAsset("expressjs"), @@ -43,6 +51,22 @@ export class ServiceConnectExpress extends Construct { ], }); + /** + * configuring a service for use with ECS Service Connect requires adding + * configuration under `serviceConnectConfiguration` + * + * `namespace` defines which namespace to connect to + * + * `services` defines the list of things to register -- see how the + * `portMappingName` configured in our task definition is wired + * up for service discovery using `discoveryName` ("blue" or "green") + * + * end result: "blue.serviceconnect" available via service discovery + * + * this configuration is used to configure an Envoy proxy that is injected + * into the service + */ + new FargateService(this, "Service", { cluster, taskDefinition,