merge from upstream (#7)

* Remove bucket for website backup

* Remove DNS for various things

Remove a bunch of DNS entries for stuff that was Cord-the-company
related, but isn't necessary for the service, like the Cord website,
Cord's hosted instance of Clack, etc.

* Remove PR server

The PR server isn't needed to run the service, and other customers
wprobably want to have their own way of doing things rather than
running our PR server exactly as it was run, so don't include it in
the ops distribution.

* Install pyOpenSSL before ec2instanceconnectcli

* Create secrets for all .env secrets

This makes the CloudFormation code create secret placeholders for all
of the different secrets need to generate .env.  The ones that are
just random strings are initialized to random strings, while the ones
that are external API keys are initialized to the string "INSERT API
KEY HERE".

* Fix names of secrets

* Remove check for Cord's GitHub repo

* Install Node 18 on build3

* Disable GitHub actions runner by default

* Don't require running under GitHub actions in build-on-commit.sh

* Don't attempt to push to getcord/monorepo on manual deploy

* feat: add JSON log format (#8)

* Disable stable host keys on zero by default

* Add keyPair definition for radical-ec2-key

* Don't customize app.cord.com bucket name

* Update version of Postgres to 15

---------

Co-authored-by: Adam Vartanian <[email protected]>
Co-authored-by: Daniel Vaz Gaspar <[email protected]>

Author: 3 people

ops/aws/src/radical-stack/Config.ts

@@ -39,9 +39,6 @@ export const CORD_COM_DOMAINS = [
 // the base for all other domains (eg, api., app.)
 export const PRIMARY_DOMAIN_NAME = CORD_COM_DOMAINS[0];
 
-// Web site domain name
-export const WEB_SITE_DOMAIN = 'cord.com';
-
 // domains for which we set up gmail
 export const GMAIL_DOMAINS = ['cord.com', 'cord.so', 'getradical.co'];
 

ops/aws/src/radical-stack/acm/dev.cord.com.ts

@@ -38,6 +38,7 @@ import { enableEc2InstanceConnect } from 'ops/aws/src/radical-stack/ec2/common.t
 import { fileUploadsBucket } from 'ops/aws/src/radical-stack/s3/fileUploads.ts';
 import { AWS_ACCOUNT, LOADTEST_TIER_ENABLED } from 'ops/aws/src/Config.ts';
 import { AWS_REGION } from 'ops/aws/src/radical-stack/Config.ts';
+import { ec2KeyPair } from 'ops/aws/src/radical-stack/ec2/keyPair.ts';
 
 const userData = (service: 'server' | 'asyncWorker', tier: Tier) => {
   const script = EC2.UserData.forLinux();
@@ -94,7 +95,7 @@ function makeServerASG(tier: Tier) {
       securityGroup: securityGroups[tier](),
       userData: userData('server', tier),
       ...Config.SERVER_AUTOSCALING_CAPACITY[tier],
-      keyName: 'radical-ec2-key',
+      keyName: ec2KeyPair().keyName,
       instanceMonitoring: autoScaling.Monitoring.DETAILED,
       healthCheck: autoScaling.HealthCheck.elb({ grace: Duration.minutes(5) }),
       notifications: [
@@ -181,7 +182,7 @@ function makeAsyncASG(tier: Tier) {
       userData: userData('asyncWorker', tier),
       minCapacity: 1,
       maxCapacity: 1,
-      keyName: 'radical-ec2-key',
+      keyName: ec2KeyPair().keyName,
       requireImdsv2: true,
     },
   );

ops/aws/src/radical-stack/ec2/autoScalingGroup.ts

@@ -25,8 +25,12 @@ import { opsNotificationTopic } from 'ops/aws/src/radical-stack/sns/topics.ts';
 import { privateSubnets } from 'ops/aws/src/radical-stack/ec2/privateSubnets.ts';
 import { basicAgentConfig } from 'ops/aws/config/cloudwatch-agent/config.ts';
 import { AWS_ACCOUNT } from 'ops/aws/src/Config.ts';
+import { ec2KeyPair } from 'ops/aws/src/radical-stack/ec2/keyPair.ts';
 
 export const hostname = 'build3';
+// Whether to install the services on the machine that allow it to operate as a
+// runner for GitHub actions, local tests, etc.
+export const INCLUDE_GITHUB_RUNNER = false;
 
 const availabilityZone = `${AWS_REGION}b`;
 const packages: string[] = [
@@ -77,6 +81,16 @@ export const build3Instance = define(() => {
     'mkdir -p /opt/aws/bin',
     'pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz',
     'ln -s /usr/local/bin/cfn-* /opt/aws/bin',
+
+    // Add NodeSource repo so we get Node 18.x, which is what we use in
+    // production, instead of whatever Ubuntu's default is at this time.
+    // https://github.com/nodesource/distributions#installation-instructions
+    'apt-get install --no-install-recommends -y ca-certificates curl gnupg',
+    'mkdir -p /etc/apt/keyrings',
+    'curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg',
+    'echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list',
+    'apt-get update',
+    'apt-get install -y nodejs',
   );
 
   const instance = new EC2.Instance(
@@ -106,7 +120,7 @@ export const build3Instance = define(() => {
       ],
       securityGroup: build3SecurityGroup(),
       vpcSubnets: { subnets: privateSubnets() },
-      keyName: 'radical-ec2-key',
+      keyName: ec2KeyPair().keyName,
       userData,
       userDataCausesReplacement: false,
     },
@@ -169,7 +183,7 @@ export const build3Instance = define(() => {
         'dockerCredHelper',
         'dockerPruneCronJob',
         'testDatabase',
-        'githubActionsRunner',
+        ...(INCLUDE_GITHUB_RUNNER ? ['githubActionsRunner'] : []),
       ],
     },
     configs: {
@@ -231,44 +245,48 @@ export const build3Instance = define(() => {
           },
         ),
       ]),
-      githubActionsRunner: new EC2.InitConfig([
-        EC2.InitFile.fromAsset(
-          `/etc/docker/compose/github-actions-runner/docker-compose.yml`,
-          `config/build3/github-actions-runner/docker-compose.yml`,
-        ),
-        EC2.InitFile.fromString(
-          '/lib/systemd/system/github-actions-runner.service',
-          [
-            '[Unit]\n',
-            'Description=GitHub Actions Runner\n',
-            'Requires=docker.service\n\n',
-            'After=docker.service\n',
-            '[Service]\n',
-            'Type=oneshot\n',
-            'RemainAfterExit=true\n',
-            'TimeoutStartSec=5m\n',
-            'WorkingDirectory=/etc/docker/compose/github-actions-runner\n',
-            'ExecStart=/usr/bin/docker-compose up -d --remove-orphans --scale runner=6\n',
-            'ExecStop=/usr/bin/docker-compose down\n',
-            'Restart=on-failure\n',
-            'RestartSec=5\n',
-            '[Install]\n',
-            'WantedBy=multi-user.target',
-          ].join(''),
-        ),
-        EC2.InitFile.fromString(
-          '/etc/cron.d/restart-github-actions-runner',
-          '# Restart every morning at 7am UTC\n' +
-            '0 7 * * * root ' +
-            `docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/github-actions-runner:latest && ` +
-            'systemctl restart github-actions-runner\n',
-        ),
-        EC2.InitCommand.shellCommand(
-          'systemctl daemon-reload && ' +
-            'systemctl enable github-actions-runner && ' +
-            'systemctl restart github-actions-runner',
-        ),
-      ]),
+      ...(INCLUDE_GITHUB_RUNNER
+        ? {
+            githubActionsRunner: new EC2.InitConfig([
+              EC2.InitFile.fromAsset(
+                `/etc/docker/compose/github-actions-runner/docker-compose.yml`,
+                `config/build3/github-actions-runner/docker-compose.yml`,
+              ),
+              EC2.InitFile.fromString(
+                '/lib/systemd/system/github-actions-runner.service',
+                [
+                  '[Unit]\n',
+                  'Description=GitHub Actions Runner\n',
+                  'Requires=docker.service\n\n',
+                  'After=docker.service\n',
+                  '[Service]\n',
+                  'Type=oneshot\n',
+                  'RemainAfterExit=true\n',
+                  'TimeoutStartSec=5m\n',
+                  'WorkingDirectory=/etc/docker/compose/github-actions-runner\n',
+                  'ExecStart=/usr/bin/docker-compose up -d --remove-orphans --scale runner=6\n',
+                  'ExecStop=/usr/bin/docker-compose down\n',
+                  'Restart=on-failure\n',
+                  'RestartSec=5\n',
+                  '[Install]\n',
+                  'WantedBy=multi-user.target',
+                ].join(''),
+              ),
+              EC2.InitFile.fromString(
+                '/etc/cron.d/restart-github-actions-runner',
+                '# Restart every morning at 7am UTC\n' +
+                  '0 7 * * * root ' +
+                  `docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/github-actions-runner:latest && ` +
+                  'systemctl restart github-actions-runner\n',
+              ),
+              EC2.InitCommand.shellCommand(
+                'systemctl daemon-reload && ' +
+                  'systemctl enable github-actions-runner && ' +
+                  'systemctl restart github-actions-runner',
+              ),
+            ]),
+          }
+        : {}),
       testDatabase: new EC2.InitConfig([
         EC2.InitFile.fromAsset(
           `/etc/docker/compose/test-db/docker-compose.yml`,

ops/aws/src/radical-stack/ec2/build3.ts

@@ -25,7 +25,11 @@ import { radicalStack } from 'ops/aws/src/radical-stack/stack.ts';
 import { vanta } from 'ops/aws/src/radical-stack/vanta.ts';
 import { basicAgentConfig } from 'ops/aws/config/cloudwatch-agent/config.ts';
 import { AWS_ACCOUNT } from 'ops/aws/src/Config.ts';
-import { AWS_REGION, S3_BUCKET_PREFIX } from 'ops/aws/src/radical-stack/Config.ts';
+import {
+  AWS_REGION,
+  S3_BUCKET_PREFIX,
+} from 'ops/aws/src/radical-stack/Config.ts';
+import { ec2KeyPair } from 'ops/aws/src/radical-stack/ec2/keyPair.ts';
 
 const stack = define(() => new NestedStack(radicalStack(), 'stack-e2eTest'));
 
@@ -94,7 +98,7 @@ const e2eTestInstance = define(() => {
     userData,
     userDataCausesReplacement: true,
     securityGroup: e2eTestSecurityGroup(),
-    keyName: 'radical-ec2-key',
+    keyName: ec2KeyPair().keyName,
     requireImdsv2: true,
   });
   vanta(instance, 'EC2 instance running automated e2e tests', {

ops/aws/src/radical-stack/ec2/e2eTest.ts

@@ -0,0 +1,9 @@
+import { aws_ec2 as EC2 } from 'aws-cdk-lib';
+import { define } from 'ops/aws/src/common.ts';
+import { radicalStack } from 'ops/aws/src/radical-stack/stack.ts';
+
+export const ec2KeyPair = define(() => {
+  return new EC2.CfnKeyPair(radicalStack(), 'radical-ec2-key', {
+    keyName: 'radical-ec2-key',
+  });
+});

ops/aws/src/radical-stack/ec2/keyPair.ts

@@ -28,8 +28,6 @@ import {
   PRIMARY_DOMAIN_NAME,
 } from 'ops/aws/src/radical-stack/Config.ts';
 import { LOADTEST_TIER_ENABLED } from 'ops/aws/src/Config.ts';
-import { prServerInstance } from 'ops/aws/src/radical-stack/ec2/prServer.ts';
-import { devCordComCertificate } from 'ops/aws/src/radical-stack/acm/dev.cord.com.ts';
 
 export const loadBalancer = define(() => {
   const lb = new elbv2.ApplicationLoadBalancer(
@@ -90,7 +88,6 @@ export const loadBalancer = define(() => {
       cordComCertificate(),
       stagingCordComCertificate(),
       loadtestCordComCertificate(),
-      devCordComCertificate(),
     ],
     sslPolicy: elbv2.SslPolicy.RECOMMENDED_TLS,
   });
@@ -225,13 +222,6 @@ export const loadBalancer = define(() => {
     });
   }
 
-  addAction(
-    'prServer',
-    50,
-    [`*.dev.${PRIMARY_DOMAIN_NAME}`],
-    prServerTargetGroup(),
-  );
-
   lb.logAccessLogs(elbLogsBucket());
 
   return lb;
@@ -259,17 +249,6 @@ export const oncallTargetGroup = define(
     }),
 );
 
-export const prServerTargetGroup = define(
-  () =>
-    new elbv2.ApplicationTargetGroup(radicalStack(), 'prServerTargetGroup', {
-      targetGroupName: 'prServer',
-      protocol: elbv2.ApplicationProtocol.HTTP,
-      port: 8081,
-      targets: [new elbv2t.InstanceTarget(prServerInstance(), 8081)],
-      vpc: defaultVpc(),
-    }),
-);
-
 export const serverAPITargetGroups = defineForEachTier(
   makeServerTargetGroup('API', 8161),
 );

ops/aws/src/radical-stack/ec2/loadBalancers.ts

@@ -30,6 +30,7 @@ import {
 } from 'ops/aws/src/radical-stack/ec2/common.ts';
 import { AWS_ACCOUNT } from 'ops/aws/src/Config.ts';
 import { AWS_REGION } from 'ops/aws/src/radical-stack/Config.ts';
+import { ec2KeyPair } from 'ops/aws/src/radical-stack/ec2/keyPair.ts';
 
 const availabilityZone = `${AWS_REGION}a`;
 
@@ -80,7 +81,7 @@ export const monitoringInstance = define(() => {
     userData,
     userDataCausesReplacement: true,
     securityGroup: monitoringSecurityGroup(),
-    keyName: 'radical-ec2-key',
+    keyName: ec2KeyPair().keyName,
     requireImdsv2: true,
   });
   vanta(